diff --git a/src/ratatoskr/tts.py b/src/ratatoskr/tts.py new file mode 100644 index 0000000..2691b1e --- /dev/null +++ b/src/ratatoskr/tts.py @@ -0,0 +1,150 @@ +"""Zonos-gateway TTS client + PAD→emotion-dial mapping. + +Slice 2 of docs/contracts/donut_voiced_interview.contract.md. This module is the +SINGLE swap seam for voice synthesis: the `/api/tts` route in web/server.py is +its only caller. Direct coupling to the Zonos gateway (DEC-1) buys the emotion +dials that the swappable `ext-tts` LiteLLM alias drops — the whole point is +affect-driven voice (DEC-7). If we ever move off Zonos, this is the swap point. + +Foot-guns (verified live 2026-08-02): + - DEC-3: response_format is ALWAYS "wav". `mp3`/`opus` are accepted but the + gateway silently returns mislabeled PCM (no encoder wired) — never request them. + - Use the gateway :8890, NOT the engine :1920 (rep-penalty bug pads silence). +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass + +import httpx + +# DEC-1: the Zonos gateway (irv-ml1 :8890) — NOT the engine :1920, NOT the +# swappable ext-tts alias (which drops the emotion dials). Overridable per +# deployment via app.state.tts_url (RATATOSKR_TTS_URL) — the swap seam + tests. +ZONOS_TTS_URL = "http://10.100.79.3:8890/v1/audio/speech" + +# DEC-7: fixed emotion strength when PAD-driven (tunable; the gateway scales the +# valence/arousal push by this). +_DEFAULT_EMOTION_STRENGTH = 1.0 + + +class TtsUnavailable(Exception): + """The Zonos gateway failed, was unreachable, or returned a non-wav body. + + The caller degrades (INV-TTS-4): logs + skips audio; the turn/transcript is + never blocked or failed on a synthesis error. + """ + + def __init__(self, message: str = "", *, status: int = 0) -> None: + super().__init__(message) + self.status = status + self.message = message + + +@dataclass(frozen=True) +class PadState: + """Live PAD read off the affect_update SSE `current` snapshot (the console + already consumes it, DEC-7). `dominance` is carried for completeness but + unused by the emotion dials — Zonos exposes valence + arousal only.""" + + pleasure: float + arousal: float + dominance: float = 0.0 + + @classmethod + def from_obj(cls, obj: object) -> PadState | None: + """Open-world parse of the browser-sent `pad`. Returns None on a missing + or malformed value (pad_to_dials then degrades to a neutral read). Never + raises — the wire is untrusted (INV-TTS-4 / degrade-not-crash).""" + if not isinstance(obj, Mapping): + return None + try: + return cls( + pleasure=float(obj["pleasure"]), + arousal=float(obj["arousal"]), + dominance=float(obj.get("dominance", 0.0)), + ) + except (KeyError, TypeError, ValueError): + return None + + +@dataclass(frozen=True) +class EmotionDials: + """Zonos emotion dials (DEC-7). Fields mirror the gateway's /v1/dials surface + (verified live 2026-08-02). `emotion_enabled=False` is a neutral read: the + other fields are omitted from the POST body, so the gateway uses its default + voice emotion.""" + + emotion_enabled: bool = False + emotion_valence: float = 0.0 + emotion_arousal: float = 0.0 + emotion_strength: float = _DEFAULT_EMOTION_STRENGTH + + def to_body(self) -> dict: + """The dial fields for the gateway POST body. Emitted ONLY when enabled; + a neutral read contributes nothing (the gateway falls to its default).""" + if not self.emotion_enabled: + return {} + return { + "emotion_enabled": True, + "emotion_valence": self.emotion_valence, + "emotion_arousal": self.emotion_arousal, + "emotion_strength": self.emotion_strength, + } + + +def _clamp(x: float, lo: float, hi: float) -> float: + """Clamp to [lo, hi], NaN-safe: a NaN axis (a dead/malformed signal) → 0.0 + rather than propagating through max/min into the gateway.""" + if x != x: # NaN + return 0.0 + return max(lo, min(hi, x)) + + +def pad_to_dials(pad: PadState | None) -> EmotionDials: + """Map live PAD → Zonos emotion dials (DEC-7 / FN pad_to_dials). + + Total: any input (None / finite / out-of-range / NaN / inf) → valid dials, + never raises. None/absent PAD → a neutral read (emotion_enabled=False).""" + if pad is None: + return EmotionDials(emotion_enabled=False) + return EmotionDials( + emotion_enabled=True, + emotion_valence=_clamp(pad.pleasure, -1.0, 1.0), + emotion_arousal=_clamp(pad.arousal, -1.0, 1.0), + emotion_strength=_DEFAULT_EMOTION_STRENGTH, + ) + + +async def tts_synthesize( + text: str, + *, + voice: str, + dials: EmotionDials, + client: httpx.AsyncClient, + url: str = ZONOS_TTS_URL, +) -> bytes: + """POST {input, voice, response_format:"wav", **dials} to the Zonos gateway; + return 16-bit RIFF/WAVE bytes (FN tts_synthesize). + + response_format is ALWAYS "wav" (DEC-3). Any non-200, transport failure, or + non-wav body → TtsUnavailable — the caller degrades (INV-TTS-4). + """ + assert text, "tts_synthesize: text must be non-empty (the endpoint guards this)" + body = {"input": text, "voice": voice, "response_format": "wav", **dials.to_body()} + try: + resp = await client.post(url, json=body) + except httpx.RequestError as exc: + raise TtsUnavailable(f"gateway transport failure: {exc}") from exc + if resp.status_code != 200: + raise TtsUnavailable( + f"gateway status {resp.status_code}", status=resp.status_code + ) + data = resp.content + # DEC-3 guard: the gateway MUST return RIFF/WAVE. A non-wav 200 (an error + # page, or the mislabeled-PCM mp3/opus trap) is treated as unavailable — + # played garbage is worse than silence. + if data[:4] != b"RIFF": + raise TtsUnavailable("gateway returned a non-wav body") + return data diff --git a/src/ratatoskr/web/entrypoint.py b/src/ratatoskr/web/entrypoint.py index 5b3e9ac..e2ed7bb 100644 --- a/src/ratatoskr/web/entrypoint.py +++ b/src/ratatoskr/web/entrypoint.py @@ -80,6 +80,11 @@ def main(argv: list[str] | None = None) -> int: # never receives the key, only the session-filtered result. admin_key = os.environ.get("RATATOSKR_ADMIN_API_KEY") + # Auto-TTS (slice 2): the Zonos gateway URL. Defaults to the direct gateway + # (DEC-1) inside the server; override here only to point at a different synth + # host (the swap seam). None → the server's ZONOS_TTS_URL default. + tts_url = os.environ.get("RATATOSKR_TTS_URL") + # INV-001: lazy import. Users without [web] extras get a clean hint # instead of a raw ImportError. Scoped narrowly to the OPTIONAL # extras (starlette / uvicorn) so a real import bug inside a @@ -100,10 +105,11 @@ def main(argv: list[str] | None = None) -> int: # Baseline deps + own modules — a failure here is a real bug, not a # missing-extras condition; let it propagate. import httpx + from ratatoskr.cli import USER_AGENT from ratatoskr.web.server import create_app - def client_factory() -> "httpx.AsyncClient": + def client_factory() -> httpx.AsyncClient: return httpx.AsyncClient( base_url=server_url, headers={ @@ -121,6 +127,7 @@ def main(argv: list[str] | None = None) -> int: affect_read_url=affect_read_url, memory_read_url=memory_read_url, admin_key=admin_key, + tts_url=tts_url, ) # Boot banner to stderr (so stdout stays clean for piping). diff --git a/src/ratatoskr/web/server.py b/src/ratatoskr/web/server.py index 3819a8e..3c3adf7 100644 --- a/src/ratatoskr/web/server.py +++ b/src/ratatoskr/web/server.py @@ -64,6 +64,13 @@ from ratatoskr.sse_client import ( SseConnectionDropped, TurnIdFlip, ) +from ratatoskr.tts import ( + ZONOS_TTS_URL, + PadState, + TtsUnavailable, + pad_to_dials, + tts_synthesize, +) def _wt_client( @@ -529,6 +536,48 @@ async def _memory_chunks_endpoint(request: Request) -> JSONResponse: return JSONResponse(r.json(), status_code=r.status_code) +# Per-character voice map (DEC-8): interview characters resolve to their registered +# Zonos voice; everything else falls to the gateway default. Case-folded gateway-side. +_TTS_VOICE_MAP = {"ratatoskr:donut": "donut"} +_TTS_DEFAULT_VOICE = "Cora" + + +async def _tts_endpoint(request: Request) -> Response: + """POST /api/tts {text, agent_id?, pad?} → audio/wav (FN tts_endpoint, slice 2). + + Server-side proxy to the Zonos gateway (DEC-4 / INV-TTS-1: the gateway host + never reaches the browser). Voice resolves per-character (DEC-8); emotion dials + map the browser-sent live PAD (DEC-7 — the affect the persona pane already shows). + Serialized one-synth-at-a-time (DEC-5 / INV-TTS-3 — the gateway shares one 3090). + A gateway failure / non-wav body degrades to 503 (INV-TTS-4: the client skips + playback; the turn/transcript is unaffected).""" + try: + body = await request.json() + except (json.JSONDecodeError, ValueError, TypeError): + body = None + text = body.get("text") if isinstance(body, dict) else None + if not text or not isinstance(text, str): + return JSONResponse({"error_code": "missing_text"}, status_code=400) + agent_id = body.get("agent_id") if isinstance(body, dict) else None + voice = _TTS_VOICE_MAP.get(agent_id, _TTS_DEFAULT_VOICE) + pad_obj = body.get("pad") if isinstance(body, dict) else None + dials = pad_to_dials(PadState.from_obj(pad_obj)) + + tts_url = request.app.state.tts_url + lock = request.app.state.tts_lock + try: + # DEC-5: serialize — a new turn's synth waits on any in-flight one (the + # client also aborts the prior request, cancelling the server task). + async with lock: + async with httpx.AsyncClient(timeout=httpx.Timeout(120.0)) as client: + wav = await tts_synthesize( + text, voice=voice, dials=dials, client=client, url=tts_url + ) + except TtsUnavailable: + return JSONResponse({"error_code": "tts_unavailable"}, status_code=503) + return Response(wav, media_type="audio/wav") + + async def _session_tools_endpoint(request: Request) -> JSONResponse: """GET /api/sessions/{session_id}/tools → owner-scoped tool inventory (spec #183). @@ -673,6 +722,7 @@ def create_app( affect_read_url: str | None = None, memory_read_url: str | None = None, admin_key: str | None = None, + tts_url: str | None = None, ) -> Starlette: """Construct the Starlette app — wire routes + state per FN create_app. @@ -739,6 +789,7 @@ def create_app( Route("/api/sessions/{session_id}/messages", _session_messages_endpoint), Route("/api/sessions/{session_id}/bifrost", _session_bifrost_endpoint), Route("/api/admin/events", _admin_events_endpoint), + Route("/api/tts", _tts_endpoint, methods=["POST"]), Route("/api/turns/{session_id}", _submit_turn_endpoint, methods=["POST"]), Route("/api/turns/{session_id}/stream", _stream_turn_endpoint), Route("/api/turns/{session_id}/cancel", _cancel_turn_endpoint, methods=["POST"]), @@ -762,6 +813,12 @@ def create_app( # SERVER-HELD (RATATOSKR_ADMIN_API_KEY) and never reaches the browser — the # server proxies admin-scoped reads and forwards only the session-filtered result. app.state.admin_key = admin_key + # Auto-TTS (slice 2): the Zonos gateway URL is SERVER-HELD config — the host + # never reaches the browser (DEC-4 / INV-TTS-1). Defaults to the direct gateway + # (DEC-1); overridable via RATATOSKR_TTS_URL (the swap seam). The lock serializes + # one synth at a time so concurrent turns don't contend the shared 3090 (DEC-5). + app.state.tts_url = tts_url or ZONOS_TTS_URL + app.state.tts_lock = asyncio.Lock() # INV-002: turn registry is in-process memory, keyed (session_id, turn_id) app.state.turn_registry = {} return app diff --git a/src/ratatoskr/web/static/index.html b/src/ratatoskr/web/static/index.html index 2ead4d1..b17f1b2 100644 --- a/src/ratatoskr/web/static/index.html +++ b/src/ratatoskr/web/static/index.html @@ -226,6 +226,12 @@ kbd { #theme-toggle .i-moon { display: none; } body[data-theme="light"] #theme-toggle .i-sun { display: none; } body[data-theme="light"] #theme-toggle .i-moon { display: inline; } +#tts-toggle { width: 30px; height: 30px; display: flex; align-items: center; justify-content: center; background: transparent; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); color: var(--fg-muted); cursor: pointer; flex: 0 0 auto; } +#tts-toggle:hover { background: var(--bg-2); color: var(--fg-0); border-color: var(--border-default); } +#tts-toggle.on { color: var(--aus-bright-cyan); border-color: var(--border-default); } +#tts-toggle .i-wave { display: none; } +#tts-toggle.on .i-wave { display: inline; } +#tts-toggle.on .i-mute { display: none; } #conn { flex: 0 0 auto; margin: 0 14px 14px; padding: 9px 12px; @@ -573,6 +579,10 @@ body.cot-hidden #cot-toggle { border-color: rgba(66,220,209,0.55); color: var(--