esh-pfi-infrastructure/stacks/searxng/conf/searxng-settings.yml

# SearXNG — PFI fleet meta-search. Deployed on nh3-docker (10.100.50.40:9996).
#
# ⚠ WHY NH3 AND NOT THE COLO. Measured 2026-09-03:
#     ana-docker egress  38.120.12.42   (datacenter) -> DuckDuckGo + Startpage CAPTCHA
#     nh3-docker egress  70.230.226.88  (residential) -> no CAPTCHA
#   Search engines gate datacenter ranges. Same reason the fleet keeps a
#   residential SOCKS5 egress proxy on nh3-dev for yt-dlp. Running the search
#   aggregator from a residential-egress site removes the problem at the source
#   rather than proxying around it.
#   UPDATE 2026-09-18: residential alone no longer sufficient — DDG + Startpage
#   now captcha even here. Tier A fix: scraped families removed, API-backed and
#   aggregator-tolerant engines kept. If Brave results are wanted, get a free
#   Brave Search API key and set `api_key` on the `braveapi` engine (inactive,
#   takes priority=0 when keyed) instead of returning to scraping.

use_default_settings:
  engines:
    remove:
      # Onion engines: no Tor proxy is configured here, so they only ever
      # contribute timeouts.
      - ahmia
      - torch
      # ⚠ Removal keys must match the engine's REAL name, spaces and all.
      # `karmasearch.videos` (dotted) did NOT match on the old instance and the
      # engine kept appearing in unresponsive_engines despite being "removed".
      # The name is "karmasearch videos".
      - karmasearch
      - karmasearch videos
      # Tier A (v4, 2026-09-18): scraped engine families that CAPTCHA/challenge
      # this egress IP. Measured 3/3 queries: duckduckgo -> CAPTCHA, startpage
      # -> Suspended: CAPTCHA, brave -> Suspended: too many requests. The whole
      # family goes: images/videos/news siblings share the same scraper and
      # network. API-backed engines never see captchas.
      - brave
      - brave.images
      - brave.videos
      - brave.news
      - duckduckgo
      - duckduckgo images
      - duckduckgo videos
      - duckduckgo news
      - startpage
      - startpage news
      - startpage images
      # Wikidata (an API engine, not scraped): Wikimedia answers this egress
      # with 403 (measured direct from the host, 6/6 searxng probes + a
      # manual sparql probe). Only feeds HTML infoboxes; keeping it meant a
      # permanent 'Suspended' entry in every response.
      - wikidata

# Breadth note (v2 experiment): mojeek was re-enabled and is ALSO 403'd from
# this egress IP (3/3 probes + direct 403 from the host, with and without a
# browser UA). Re-adding breadth now needs API keys, not scraping: braveapi
# (free Brave Search API tier) and/or marginalia (free key) — both already
# present in the defaults, awaiting an api_key.
#
# v4 (2026-09-18): braveapi keyed. The key lives in the container env
# (compose .env, vault: nh3-docker/searxng-brave-api-key) and enters this file
# via SearXNG's !ENV tag — no secret in a git-tracked file. If the key line
# ever resolves empty, the engine silently drops to inactive: no error, just
# missing results. Check `unresponsive_engines`/`/config` if Brave vanishes.
#
# marginalia uses the public key (rate-limited but keyless-tier, safe to
# hardcode). A GLM web-search engine is possible as a custom JSON engine but
# needs a z.ai/bigmodel search key — not in the vault as of today.
engines:
  # ⚠ THE KEY BELOW IS COMMITTED ON PURPOSE. Operator decision, 2026-09-18:
  # this is a free-tier Brave Search API key on a rate-limited service of
  # marginal value — "if the service is useless, so is the key" — so it is not
  # worth the machinery that keeping it out of git would cost. Do not treat
  # this as licence for other credentials; everything else goes in the vault
  # (this key is also at nh3-docker/searxng-brave-api-key).
  #
  # ⚠ It cannot be un-committed. Rotation means issuing a NEW key at Brave and
  # replacing this line, never rewriting history — the repo is shared and
  # other sessions commit to it.
  #
  # An earlier attempt used `api_key: !ENV SEARXNG_BRAVE_API_KEY`. This build
  # has no !ENV YAML constructor, so the file failed to parse and the container
  # crash-looped ten times with search down fleet-wide. There is no env-var
  # path into this file: the loader reads only SEARXNG_SETTINGS_PATH, and the
  # entrypoint substitutes only `ultrasecretkey`. Literal or nothing.
  - name: braveapi
    api_key: BSAa55OlyBPDjHcvyRiOcBl5USayGXb
    disabled: false
    inactive: false
  - name: marginalia
    api_key: public
    disabled: false
    inactive: false

  # ── Breadth: keyless engines that are ON by default ───────────────────
  # ⚠ THE BUG THIS FIXES. Of 55 general-category engines, only SEVEN were
  # enabled-by-default, and six of those are dictionary/translation/currency/
  # encyclopedia engines that return nothing for an ordinary web query. That
  # left `google cse` as the instance's ONLY general web engine — so one quota
  # exhaustion returned HTTP 200 with an empty results array and no error, for
  # every consumer. `disabled: false` is what puts an engine in the default
  # set; `inactive: false` only makes it selectable. The other 48 are
  # selectable-but-off, which an API client cannot change.
  #
  # Each of the five below was bang-probed on 2026-09-18 and returned real
  # results with no key: ddgw 10, bing 10, yep 20, yahoo 7, wiby 12.
  # Deliberately NOT added: mojeek, qwant, startpage, brave (scrapers — all
  # CAPTCHA or rate-limit this egress) and seznam (timeout).
  - name: duckduckgo web
    disabled: false
  - name: bing
    disabled: false
  - name: yep
    disabled: false
  - name: yahoo
    disabled: false
  - name: wiby
    disabled: false

general:
  instance_name: "SearXNG"
  instance_about_url: false
  contact_url: false
  debug: false
  # Public metrics page off — smaller attack surface on an unauthenticated
  # internal service.
  enable_metrics: false

search:
  safe_search: 0
  autocomplete: ""
  default_lang: "auto"
  # `json` is what makes this usable as a tool rather than only a web page.
  # Removing it breaks every non-browser consumer, including Claude sessions.
  formats:
    - html
    - json
  # 3s is too tight for slower engines; 8s covers them without hanging the UI.
  request_timeout: 8.0
  # Ban an engine only briefly when it raises suspended-time. The default 86400
  # means one bad afternoon silences an engine for a day.
  ban_time_on_fail: 60
  max_ban_time_on_fail: 600

server:
  # secret_key comes from SEARXNG_SECRET in the environment — never hardcode it
  # here. Generated + vaulted at nh3-docker/searxng-secret.
  bind_address: "0.0.0.0"
  port: 8080
  # Enable ONLY with a limiter.toml AND a proxy that forwards X-Real-IP;
  # otherwise it logs "X-Forwarded-For nor X-Real-IP header is set!" forever.
  limiter: false
  public_instance: false
  # ⚠ Kept in sync with BASE_URL in compose.yaml. The old instance still said
  # `https://searxng.pfi.local/` here — a name retired on 2026-08-19 — while the
  # environment said something else. The env wins, so nothing broke, and the
  # file quietly lied to everyone who read it.
  base_url: "http://10.100.50.40:9996/"
  method: "GET"
  compression: true
  image_proxy: false

# Outgoing pool — low-traffic private instance.
outgoing:
  request_timeout: 6.0
  max_request_timeout: 12.0
  pool_connections: 100
  pool_maxsize: 20
  enable_http2: true
  # No proxy needed: this host already egresses residentially (see header).
  # If that ever changes, the fleet's NH3 SOCKS5 proxy is the fallback:
  #   proxies:
  #     all://:
  #       - socks5h://10.100.10.50:1080