From 41359eaff96d416aab00df64e1fe975a222bb2ad Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Tue, 11 Aug 2026 16:02:30 -0700 Subject: [PATCH] feat(secrets-broker): secret CLI (bw-backed fleet credential store) + contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit secret put/get/list/backfill over Vaultwarden via the bw CLI. Items land in the infra-ops org's Default collection (visible to the operator's primary account via org share), organised by folder + // naming; text in the note, binary base64'd into a hidden field; sha256 + source metadata fields; idempotent upsert keyed by name. Auth bootstraps from ~/.config/secrets-broker/bootstrap.env (0600, apikey login + master-password unlock, per-invocation session). Verified live end-to-end (create/upsert/get-note/get-field/list). Contract updated: bw replaces rbw (rbw register 400'd undebuggably despite valid creds). Known limitation: bw-subprocess-per-op is ~3s/call → ~15-25s/command; too slow for a fleet-scale backfill. Next: a bw serve broker (fast + central-cred fleet model). --- services/secrets-broker/secret | 256 ++++++++++++++++++ .../secrets-broker/secrets-broker.contract.md | 159 +++++++++++ 2 files changed, 415 insertions(+) create mode 100755 services/secrets-broker/secret create mode 100644 services/secrets-broker/secrets-broker.contract.md diff --git a/services/secrets-broker/secret b/services/secrets-broker/secret new file mode 100755 index 0000000..442d3f1 --- /dev/null +++ b/services/secrets-broker/secret @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""secret — fleet credential store & retrieve over Vaultwarden (bw-backed). + +Stores each secret as an item in the `infra-ops` org's Default collection (so the +operator's primary account, an org member, sees it too), organised by folder + +a `//` name convention. Text secrets (env files, tokens) live +in the item note; binary secrets (keys/certs) are base64'd into a hidden field. + +Auth bootstraps from ~/.config/secrets-broker/bootstrap.env (0600): apikey login ++ master-password unlock -> per-invocation session. Secret VALUES are never printed +except by an explicit `get`. + +Commands: + secret put (--file P | --stdin) [--folder C] [--field k=v]... + secret get [--field F] [--file OUT] + secret list [--prefix P] + secret backfill --host H [--dry-run] # enumerate a host's .env/env.sh, upsert +""" +import argparse +import base64 +import hashlib +import json +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path + +BW = os.environ.get("BW_BIN", str(Path.home() / ".local/bin/bw")) +CFG = Path(os.environ.get("SECRET_CFG_DIR", str(Path.home() / ".config/secrets-broker"))) +BOOTSTRAP = CFG / "bootstrap.env" +SERVER = "https://vaultwarden.phasefinal.com" +ORG_ID = "d30c6b58-773c-4e39-916e-8e33ca7f8b81" +COLLECTION_ID = "b829376a-9db1-4091-a4ae-9a36132ad4c2" + + +def die(msg): + print(f"secret: {msg}", file=sys.stderr) + sys.exit(1) + + +def load_env(): + if not BOOTSTRAP.is_file(): + die(f"no bootstrap creds at {BOOTSTRAP}") + if oct(BOOTSTRAP.stat().st_mode)[-3:] not in ("600", "400"): + die(f"{BOOTSTRAP} must be 0600") + creds = {} + for line in BOOTSTRAP.read_text().splitlines(): + for k in ("BW_CLIENTID", "BW_CLIENTSECRET", "RBW_MASTER_PW"): + if line.startswith(k + "="): + creds[k] = line.split("=", 1)[1] + env = dict(os.environ) + env["BW_CLIENTID"] = creds.get("BW_CLIENTID", "") + env["BW_CLIENTSECRET"] = creds.get("BW_CLIENTSECRET", "") + env["BW_PASSWORD"] = creds.get("RBW_MASTER_PW", "") + return env + + +def bw(args, env, session=None, stdin=None, check=True): + cmd = [BW] + args + (["--session", session] if session else []) + r = subprocess.run(cmd, env=env, input=stdin, capture_output=True) + if check and r.returncode != 0: + die(f"bw {args[0]} failed: {r.stderr.decode(errors='replace').strip()}") + return r + + +def status(env): + r = subprocess.run([BW, "status"], env=env, capture_output=True) + try: + return json.loads(r.stdout.decode()).get("status", "unauthenticated") + except Exception: + return "unauthenticated" + + +def session(env): + if status(env) == "unauthenticated": + subprocess.run([BW, "config", "server", SERVER], env=env, capture_output=True) + bw(["login", "--apikey"], env) + s = bw(["unlock", "--passwordenv", "BW_PASSWORD", "--raw"], env).stdout.decode().strip() + return s or die("unlock produced no session") + + +def encode(obj, env): + return bw(["encode"], env, stdin=json.dumps(obj).encode()).stdout + + +def find(env, s, name): + r = bw(["list", "items", "--search", name], env, s) + for it in json.loads(r.stdout.decode() or "[]"): + if it.get("name") == name: + return it + return None + + +def folder_id(env, s, cls): + if not cls: + return None + for f in json.loads(bw(["list", "folders"], env, s).stdout.decode() or "[]"): + if f.get("name") == cls: + return f["id"] + r = bw(["create", "folder"], env, s, stdin=encode({"name": cls}, env)) + return json.loads(r.stdout.decode())["id"] + + +def cmd_put(a): + env = load_env() + s = session(env) + if a.file: + data = Path(a.file).read_bytes() + elif a.stdin: + data = sys.stdin.buffer.read() + else: + die("put needs --file or --stdin") + sha = hashlib.sha256(data).hexdigest() + try: + notes, binary = data.decode("utf-8"), False + except UnicodeDecodeError: + notes, binary = "", True + fields = [ + {"name": "sha256", "value": sha, "type": 0}, + {"name": "synced_at", "value": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "type": 0}, + ] + for kv in (a.field or []): + k, v = kv.split("=", 1) + fields.append({"name": k, "value": v, "type": 0}) + if binary: + fields.append({"name": "content_b64", "value": base64.b64encode(data).decode(), "type": 1}) + item = { + "type": 2, "name": a.name, "notes": notes, + "organizationId": ORG_ID, "collectionIds": [COLLECTION_ID], + "folderId": folder_id(env, s, a.folder), "fields": fields, + "secureNote": {"type": 0}, + } + existing = find(env, s, a.name) + if existing: + item["id"] = existing["id"] + bw(["edit", "item", existing["id"]], env, s, stdin=encode(item, env)) + action = "updated" + else: + bw(["create", "item"], env, s, stdin=encode(item, env)) + action = "created" + print(f"{action}: {a.name} (sha256 {sha[:12]}, {len(data)} bytes{', binary' if binary else ''})") + + +def cmd_get(a): + env = load_env() + s = session(env) + it = find(env, s, a.name) or die(f"not found: {a.name}") + full = json.loads(bw(["get", "item", it["id"]], env, s).stdout.decode()) + if a.field: + for f in full.get("fields") or []: + if f["name"] == a.field: + if a.field == "content_b64" and a.file: + Path(a.file).write_bytes(base64.b64decode(f["value"])) + os.chmod(a.file, 0o600) + return print(f"wrote {a.file} (0600)") + return print(f["value"]) + die(f"no field '{a.field}' on {a.name}") + content = full.get("notes") or "" + if a.file: + Path(a.file).write_text(content) + os.chmod(a.file, 0o600) + print(f"wrote {a.file} (0600)") + else: + sys.stdout.write(content if content.endswith("\n") else content + "\n") + + +def cmd_list(a): + env = load_env() + s = session(env) + items = json.loads(bw(["list", "items"], env, s).stdout.decode() or "[]") + n = 0 + for it in sorted(items, key=lambda x: x.get("name", "")): + name = it.get("name", "") + if a.prefix and not name.startswith(a.prefix): + continue + meta = {f["name"]: f["value"] for f in (it.get("fields") or [])} + print(f"{name}\t{meta.get('synced_at', '?')}\t{meta.get('sha256', '')[:12]}") + n += 1 + print(f"# {n} item(s)", file=sys.stderr) + + +def _host_secret_files(host): + find_cmd = (r"find /opt/docker -maxdepth 4 \( -name .env -o -name env.sh \) " + r"! -name '*.example' -type f 2>/dev/null") + r = subprocess.run(["ssh", host, find_cmd], capture_output=True, text=True) + return [p for p in r.stdout.split("\n") if p.strip()] + + +def _name_for(host, path): + parts = Path(path).parts # e.g. /opt/docker/compose/gitea/.env + stack = parts[-2] if len(parts) >= 2 else "root" + return f"{host}/{stack}/{Path(path).name}" + + +def cmd_backfill(a): + host = a.host + files = _host_secret_files(host) + if not files: + return print(f"{host}: no .env/env.sh found under /opt/docker") + env = load_env() + s = None if a.dry_run else session(env) + print(f"{host}: {len(files)} secret file(s)") + for path in files: + name = _name_for(host, path) + data = subprocess.run(["ssh", host, f"cat {shlex.quote(path)}"], capture_output=True).stdout + sha = hashlib.sha256(data).hexdigest() + if a.dry_run: + print(f" WOULD store {name}\t({len(data)} bytes, sha {sha[:12]}) <- {path}") + continue + try: + notes = data.decode("utf-8") + except UnicodeDecodeError: + notes = "" + fields = [ + {"name": "sha256", "value": sha, "type": 0}, + {"name": "source_host", "value": host, "type": 0}, + {"name": "source_path", "value": path, "type": 0}, + {"name": "synced_at", "value": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "type": 0}, + ] + item = {"type": 2, "name": name, "notes": notes, "organizationId": ORG_ID, + "collectionIds": [COLLECTION_ID], "folderId": folder_id(env, s, "hosts"), + "fields": fields, "secureNote": {"type": 0}} + existing = find(env, s, name) + if existing: + item["id"] = existing["id"] + bw(["edit", "item", existing["id"]], env, s, stdin=encode(item, env)) + verb = "updated" + else: + bw(["create", "item"], env, s, stdin=encode(item, env)) + verb = "stored " + # round-trip verify + back = json.loads(bw(["get", "item", find(env, s, name)["id"]], env, s).stdout.decode()) + ok = hashlib.sha256((back.get("notes") or "").encode()).hexdigest() == sha + print(f" {verb} {name}\t({len(data)}b) verify={'OK' if ok else 'MISMATCH'}") + + +def main(): + p = argparse.ArgumentParser(prog="secret", description="fleet credential store over Vaultwarden") + sub = p.add_subparsers(dest="cmd", required=True) + pp = sub.add_parser("put"); pp.add_argument("name") + pp.add_argument("--file"); pp.add_argument("--stdin", action="store_true") + pp.add_argument("--folder"); pp.add_argument("--field", action="append") + pp.set_defaults(fn=cmd_put) + pg = sub.add_parser("get"); pg.add_argument("name") + pg.add_argument("--field"); pg.add_argument("--file"); pg.set_defaults(fn=cmd_get) + pl = sub.add_parser("list"); pl.add_argument("--prefix"); pl.set_defaults(fn=cmd_list) + pb = sub.add_parser("backfill"); pb.add_argument("--host", required=True) + pb.add_argument("--dry-run", action="store_true"); pb.set_defaults(fn=cmd_backfill) + a = p.parse_args() + a.fn(a) + + +if __name__ == "__main__": + main() diff --git a/services/secrets-broker/secrets-broker.contract.md b/services/secrets-broker/secrets-broker.contract.md new file mode 100644 index 0000000..0b7d915 --- /dev/null +++ b/services/secrets-broker/secrets-broker.contract.md @@ -0,0 +1,159 @@ +--- +title: secrets-broker +kind: module-contract +status: draft +owner: infra-ops +created: 2026-08-11 +depends_on: + - Vaultwarden (vaultwarden.phasefinal.com, on ana-docker; DB on pfi-postgres, in the pg_dump backup set) + - rbw (Rust Bitwarden CLI; unattended unlock daemon) +--- + +# secrets-broker — fleet-wide credential registry over Vaultwarden + +## Purpose + +A single place to **store and look up any fleet secret** — env files, API tokens, +TLS certs/keys, SSH keys, DB creds, WireGuard keys — that should not live in a git +repo and is today single-copy on a host. Vaultwarden is the durable central store; +the broker is the thin programmatic surface (`secret put|get|list|backfill`) that +agents, scripts, and the operator use. Solves two problems at once: **durability** +(secrets currently backed up nowhere) and **lookup** (no canonical place to find a +credential). + +This is a **central store**, not (yet) a deploy-time source of truth: host `.env` +files stay in place; the vault is the authoritative *registry + backup*. The +pull-at-deploy evolution is explicitly out of scope for v1 (see Out of scope). + +## Architecture + +- **Store:** a **dedicated Vaultwarden service account** (`secrets-broker@…`), its + personal vault *is* the registry. (An Organization + per-host Collections is the + textbook multi-user structure, but **rbw's write path into org collections is + weak** — see the rbw capability gate — so v1 uses the service account's own vault + with a folder/name taxonomy. Org migration is a future option if granular + human ACLs are ever needed.) +- **Client:** **`bw`** (official Bitwarden CLI, installed to `~/.local` on nh3-dev). + Switched from `rbw` after rbw's `register` returned an undebuggable 400 against this + Vaultwarden despite valid creds (a direct `client_credentials` grant + both prelogin + paths return 200; rbw emits no HTTP logs). bw gives a clean unattended flow + (`login --apikey` via `BW_CLIENTID`/`BW_CLIENTSECRET` env, `unlock --passwordenv + --raw` → `BW_SESSION`) and full write support — org collections + attachments + included, which reopens the org-vs-personal-vault choice rbw had foreclosed. +- **Broker host + identity:** runs as the session user (`lkraven`) on **nh3-dev** + (this machine — where the live CC sessions + rbw live); rbw 1.15.0 is also present + on nh3-extdev. Fleet ssh + the trust boundary for the bootstrap secret. +- **Code home:** `eshpfi-management/services/secrets-broker/`. + +## Data model / taxonomy (lookup-first) + +- One **item per secret**. Item **name encodes the path** so lookup works even if + folders are unavailable: `//` (e.g. `ana-docker/gitea/.env`), + or `/` for non-host creds (`gitea/claude-bot-token`, + `wireguard/irv-ml1`, `certs/phasefinal-wildcard`). +- **Folders** (if rbw supports `add --folder` — gate) group by class: `hosts/`, + `tokens/`, `certs/`, `ssh-keys/`, `db/`, `wireguard/`, `misc/`. +- **Item type by shape:** a credential pair → Bitwarden *login* (username/password/ + uri); an env file or freeform blob → *secure note* (whole file in the note body). +- **Binary secrets** (`*.pem`, `*.key`, `*.pfx`, `acme.json`, `client_secrets.json`) + → **base64 into a note field** (rbw has no attachments). Cap ~64 KB/item; larger + keys are flagged, not silently truncated. +- Metadata fields on every item: `source_host`, `source_path`, `synced_at`, + `sha256` (of the plaintext, for drift detection without decrypting to compare). + +## CLI surface (`services/secrets-broker/secret`) + +- `secret put [--file PATH | --stdin] [--type note|login] [--field k=v]…` + — idempotent **upsert** (create if absent, else update in place, keyed by name). + Never prints the secret value. Refuses `.env.example` / template inputs. +- `secret get [--field FIELD] [--file OUT]` — fetch to stdout or a `0600` + file; base64 fields decode with `--file`. +- `secret list [--prefix P]` — names + metadata only (never values). +- `secret backfill --host H [--dry-run]` — ssh to host H, enumerate real secret + files (`.env`, `env.sh`, and the cert/key set), upsert each. `--dry-run` prints + the plan (names, sizes, would-create/would-update) and writes nothing. + +## Bootstrap secret handling + +- The service account's **email + master password + personal API key + (client_id/secret)** live in a **`0600` file** at + `~/.config/secrets-broker/bootstrap.env` on nh3-dev (session-user-owned) — mirrors + the `~/.config//` pattern used across the fleet. **The operator + provisions the account and drops these creds** (they never transit the chat; a + `0600` template is pre-staged at `bootstrap.env.template`). This is the one secret + that cannot live in the vault (secrets-zero); the crown jewel. +- bw unattended flow: `bw login --apikey` (reads `BW_CLIENTID`/`BW_CLIENTSECRET` + from env, sourced from the bootstrap file) → `bw unlock --passwordenv BW_PASSWORD + --raw` → a session token exported as `BW_SESSION` for subsequent calls. No + pinentry/daemon (the abandoned rbw shim + `.pinentry-key` marker can be removed). + +## Invariants + +1. **Idempotent upserts** — re-running `put`/`backfill` converges, never duplicates + (name is the key). +2. **Never log or echo secret values** — not to stdout, stderr, logs, or the board. + `list` and `--dry-run` show names/metadata only. +3. **`.env.example` and templates are never stored** — real secrets only. +4. **Round-trip verified** — backfill re-fetches and compares `sha256` against the + source before reporting a file "stored". +5. **Bootstrap file is `0600`**, owned by the broker identity; the broker host is + the trust boundary (the service account can *read* everything it can write — + Bitwarden has no write-only). +6. **Vault durability is a precondition** — the Vaultwarden DB is in the pg_dump set + (confirmed); v1 adds a periodic independent **encrypted vault export → restic** as + belt-and-suspenders before it becomes load-bearing. + +## Phases + +- **Phase 0 — verify (read-only): DONE.** Vault located + DB-backed-up; creds valid + (direct grant 200); rbw abandoned (register 400); bw selected. +- **Phase 1 — stand up: DONE (2026-08-11).** Operator provisioned the service account + (`infra-ops@phasefinal.com`); bw installed to `~/.local`; `bw config server` + + `login --apikey` + `unlock --passwordenv` all succeed; **create→read→delete + round-trip verified** against the live vault. Remaining Phase-1 polish: decide the + folder-vs-org taxonomy (bw reopened org) and lay it out. +- **Phase 2 — backfill.** `secret backfill --host --dry-run` for every host, + review, then real run; round-trip verify. Start with `.env`/`env.sh`, then the + cert/key set. +- **Phase 3 — CLI + keep-fresh.** Harden the `secret` CLI; a periodic re-sync + (host→vault) so rotated secrets don't drift; the encrypted export→restic job. + +## The rbw capability gate — RESOLVED (rbw 1.15.0, 2026-08-11) + +Verified against the installed `rbw 1.15.0` write surface on nh3-extdev: +- `add`/`edit` support **`--folder`**, `--uri`, username, and password + multi-line + note. **Folders work** → taxonomy is folder + name-encoded path. +- **No `--organization`/`--collection` on `add`** → rbw *cannot* create items in an + org collection. **Resolution: v1 uses the service account's own vault + folders** + (pure-rbw, honors the rust-client choice; no `bw` write-path needed). Org + + Collections stays deferred — it would force `bw` for writes for marginal + multi-user-ACL benefit the fleet store doesn't need. +- `add` is **editor-driven** (first line = password, the rest = note). The `secret + put` wrapper drives it non-interactively by setting `$EDITOR`/`$VISUAL` to a + content-supplying shim (standard rbw scripting pattern) — no human editor in the + automated path. Env-file blobs live in the note; single tokens as the password; + metadata (sha256/source) as trailing note lines read back with `get --field`. + +Consequence for human lookup: the registry is the **service account's shared vault** +— the operator looks things up by logging in *as* `secrets-broker` (or pointing rbw +at it), not via a shared collection in his personal account. Acceptable for a fleet +infra store; the org path remains the escape hatch if per-user ACLs are ever needed. + +## Out of scope (v1) + +- **Deploy-time source-of-truth** (deploys *pulling* secrets from the vault into + `.env`). This is the powerful evolution; it changes every deploy path and makes a + vault outage deploy-blocking. Revisit after the registry is proven. +- **Organization + Collections** with granular multi-user ACLs (rbw-write-limited; + future). +- **Bitwarden Secrets Manager** — Vaultwarden does not implement it; not an option. + +## Failure modes / rollback + +- Broker/vault down → lookups fail, but host `.env` are untouched and authoritative + (backup semantics), so nothing stops running or deploying. +- Bad backfill write → idempotent upsert + round-trip verify catch it; items are + versioned in Vaultwarden (restore prior). +- Bootstrap file compromise = full registry read → tight perms + broker-host trust + are the control; rotate the service account's master password + API key on + suspicion.