feat: Donut voiced-interview slice-2 — auto-TTS via the Zonos gateway
Adds affect-modulated voice to the web console: the completed assistant
response is spoken on SSE `done`, emotion-modulated by the live PAD the persona
pane already shows (DEC-7 — voice as affect OBSERVABILITY, not chat-app TTS).
- src/ratatoskr/tts.py (new): Zonos-gateway client + PAD→emotion-dial mapping.
tts_synthesize POSTs {input, voice, response_format:"wav", **dials}; wav-only
(DEC-3 — mp3/opus silently return mislabeled PCM). pad_to_dials is total
(None/NaN/out-of-range → valid dials, never raises). TtsUnavailable on any
gateway failure; the single swap seam if we ever move off Zonos.
- web/server.py: POST /api/tts proxy (DEC-4/INV-TTS-1 — the gateway host never
reaches the browser). Per-character voice map (DEC-8: ratatoskr:donut→donut),
serialize lock (DEC-5 — shared 3090), 503 degrade (INV-TTS-4).
- web/static/index.html: 🔊 toggle (opt-in, localStorage, default off,
INV-TTS-2), speak-on-done, AbortController cancel-on-new-turn (INV-TTS-3),
hidden <audio> sink; PAD read off the pane's current snapshot.
- web/entrypoint.py: RATATOSKR_TTS_URL override (the swap seam).
TDD: 17 tts unit tests + 5 endpoint tests (516 green). Live-smoked end-to-end
against the Zonos gateway (:8890): Donut voice + affect dials → 44.1kHz wav,
missing-text→400, neutral→200, gateway-fail→503.
Per docs/contracts/donut_voiced_interview.contract.md (slice 2 of 3).
This commit is contained in:
@@ -0,0 +1,150 @@
|
|||||||
|
"""Zonos-gateway TTS client + PAD→emotion-dial mapping.
|
||||||
|
|
||||||
|
Slice 2 of docs/contracts/donut_voiced_interview.contract.md. This module is the
|
||||||
|
SINGLE swap seam for voice synthesis: the `/api/tts` route in web/server.py is
|
||||||
|
its only caller. Direct coupling to the Zonos gateway (DEC-1) buys the emotion
|
||||||
|
dials that the swappable `ext-tts` LiteLLM alias drops — the whole point is
|
||||||
|
affect-driven voice (DEC-7). If we ever move off Zonos, this is the swap point.
|
||||||
|
|
||||||
|
Foot-guns (verified live 2026-08-02):
|
||||||
|
- DEC-3: response_format is ALWAYS "wav". `mp3`/`opus` are accepted but the
|
||||||
|
gateway silently returns mislabeled PCM (no encoder wired) — never request them.
|
||||||
|
- Use the gateway :8890, NOT the engine :1920 (rep-penalty bug pads silence).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
# DEC-1: the Zonos gateway (irv-ml1 :8890) — NOT the engine :1920, NOT the
|
||||||
|
# swappable ext-tts alias (which drops the emotion dials). Overridable per
|
||||||
|
# deployment via app.state.tts_url (RATATOSKR_TTS_URL) — the swap seam + tests.
|
||||||
|
ZONOS_TTS_URL = "http://10.100.79.3:8890/v1/audio/speech"
|
||||||
|
|
||||||
|
# DEC-7: fixed emotion strength when PAD-driven (tunable; the gateway scales the
|
||||||
|
# valence/arousal push by this).
|
||||||
|
_DEFAULT_EMOTION_STRENGTH = 1.0
|
||||||
|
|
||||||
|
|
||||||
|
class TtsUnavailable(Exception):
|
||||||
|
"""The Zonos gateway failed, was unreachable, or returned a non-wav body.
|
||||||
|
|
||||||
|
The caller degrades (INV-TTS-4): logs + skips audio; the turn/transcript is
|
||||||
|
never blocked or failed on a synthesis error.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, message: str = "", *, status: int = 0) -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
self.status = status
|
||||||
|
self.message = message
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PadState:
|
||||||
|
"""Live PAD read off the affect_update SSE `current` snapshot (the console
|
||||||
|
already consumes it, DEC-7). `dominance` is carried for completeness but
|
||||||
|
unused by the emotion dials — Zonos exposes valence + arousal only."""
|
||||||
|
|
||||||
|
pleasure: float
|
||||||
|
arousal: float
|
||||||
|
dominance: float = 0.0
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_obj(cls, obj: object) -> PadState | None:
|
||||||
|
"""Open-world parse of the browser-sent `pad`. Returns None on a missing
|
||||||
|
or malformed value (pad_to_dials then degrades to a neutral read). Never
|
||||||
|
raises — the wire is untrusted (INV-TTS-4 / degrade-not-crash)."""
|
||||||
|
if not isinstance(obj, Mapping):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return cls(
|
||||||
|
pleasure=float(obj["pleasure"]),
|
||||||
|
arousal=float(obj["arousal"]),
|
||||||
|
dominance=float(obj.get("dominance", 0.0)),
|
||||||
|
)
|
||||||
|
except (KeyError, TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class EmotionDials:
|
||||||
|
"""Zonos emotion dials (DEC-7). Fields mirror the gateway's /v1/dials surface
|
||||||
|
(verified live 2026-08-02). `emotion_enabled=False` is a neutral read: the
|
||||||
|
other fields are omitted from the POST body, so the gateway uses its default
|
||||||
|
voice emotion."""
|
||||||
|
|
||||||
|
emotion_enabled: bool = False
|
||||||
|
emotion_valence: float = 0.0
|
||||||
|
emotion_arousal: float = 0.0
|
||||||
|
emotion_strength: float = _DEFAULT_EMOTION_STRENGTH
|
||||||
|
|
||||||
|
def to_body(self) -> dict:
|
||||||
|
"""The dial fields for the gateway POST body. Emitted ONLY when enabled;
|
||||||
|
a neutral read contributes nothing (the gateway falls to its default)."""
|
||||||
|
if not self.emotion_enabled:
|
||||||
|
return {}
|
||||||
|
return {
|
||||||
|
"emotion_enabled": True,
|
||||||
|
"emotion_valence": self.emotion_valence,
|
||||||
|
"emotion_arousal": self.emotion_arousal,
|
||||||
|
"emotion_strength": self.emotion_strength,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _clamp(x: float, lo: float, hi: float) -> float:
|
||||||
|
"""Clamp to [lo, hi], NaN-safe: a NaN axis (a dead/malformed signal) → 0.0
|
||||||
|
rather than propagating through max/min into the gateway."""
|
||||||
|
if x != x: # NaN
|
||||||
|
return 0.0
|
||||||
|
return max(lo, min(hi, x))
|
||||||
|
|
||||||
|
|
||||||
|
def pad_to_dials(pad: PadState | None) -> EmotionDials:
|
||||||
|
"""Map live PAD → Zonos emotion dials (DEC-7 / FN pad_to_dials).
|
||||||
|
|
||||||
|
Total: any input (None / finite / out-of-range / NaN / inf) → valid dials,
|
||||||
|
never raises. None/absent PAD → a neutral read (emotion_enabled=False)."""
|
||||||
|
if pad is None:
|
||||||
|
return EmotionDials(emotion_enabled=False)
|
||||||
|
return EmotionDials(
|
||||||
|
emotion_enabled=True,
|
||||||
|
emotion_valence=_clamp(pad.pleasure, -1.0, 1.0),
|
||||||
|
emotion_arousal=_clamp(pad.arousal, -1.0, 1.0),
|
||||||
|
emotion_strength=_DEFAULT_EMOTION_STRENGTH,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def tts_synthesize(
|
||||||
|
text: str,
|
||||||
|
*,
|
||||||
|
voice: str,
|
||||||
|
dials: EmotionDials,
|
||||||
|
client: httpx.AsyncClient,
|
||||||
|
url: str = ZONOS_TTS_URL,
|
||||||
|
) -> bytes:
|
||||||
|
"""POST {input, voice, response_format:"wav", **dials} to the Zonos gateway;
|
||||||
|
return 16-bit RIFF/WAVE bytes (FN tts_synthesize).
|
||||||
|
|
||||||
|
response_format is ALWAYS "wav" (DEC-3). Any non-200, transport failure, or
|
||||||
|
non-wav body → TtsUnavailable — the caller degrades (INV-TTS-4).
|
||||||
|
"""
|
||||||
|
assert text, "tts_synthesize: text must be non-empty (the endpoint guards this)"
|
||||||
|
body = {"input": text, "voice": voice, "response_format": "wav", **dials.to_body()}
|
||||||
|
try:
|
||||||
|
resp = await client.post(url, json=body)
|
||||||
|
except httpx.RequestError as exc:
|
||||||
|
raise TtsUnavailable(f"gateway transport failure: {exc}") from exc
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise TtsUnavailable(
|
||||||
|
f"gateway status {resp.status_code}", status=resp.status_code
|
||||||
|
)
|
||||||
|
data = resp.content
|
||||||
|
# DEC-3 guard: the gateway MUST return RIFF/WAVE. A non-wav 200 (an error
|
||||||
|
# page, or the mislabeled-PCM mp3/opus trap) is treated as unavailable —
|
||||||
|
# played garbage is worse than silence.
|
||||||
|
if data[:4] != b"RIFF":
|
||||||
|
raise TtsUnavailable("gateway returned a non-wav body")
|
||||||
|
return data
|
||||||
@@ -80,6 +80,11 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
# never receives the key, only the session-filtered result.
|
# never receives the key, only the session-filtered result.
|
||||||
admin_key = os.environ.get("RATATOSKR_ADMIN_API_KEY")
|
admin_key = os.environ.get("RATATOSKR_ADMIN_API_KEY")
|
||||||
|
|
||||||
|
# Auto-TTS (slice 2): the Zonos gateway URL. Defaults to the direct gateway
|
||||||
|
# (DEC-1) inside the server; override here only to point at a different synth
|
||||||
|
# host (the swap seam). None → the server's ZONOS_TTS_URL default.
|
||||||
|
tts_url = os.environ.get("RATATOSKR_TTS_URL")
|
||||||
|
|
||||||
# INV-001: lazy import. Users without [web] extras get a clean hint
|
# INV-001: lazy import. Users without [web] extras get a clean hint
|
||||||
# instead of a raw ImportError. Scoped narrowly to the OPTIONAL
|
# instead of a raw ImportError. Scoped narrowly to the OPTIONAL
|
||||||
# extras (starlette / uvicorn) so a real import bug inside a
|
# extras (starlette / uvicorn) so a real import bug inside a
|
||||||
@@ -100,10 +105,11 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
# Baseline deps + own modules — a failure here is a real bug, not a
|
# Baseline deps + own modules — a failure here is a real bug, not a
|
||||||
# missing-extras condition; let it propagate.
|
# missing-extras condition; let it propagate.
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from ratatoskr.cli import USER_AGENT
|
from ratatoskr.cli import USER_AGENT
|
||||||
from ratatoskr.web.server import create_app
|
from ratatoskr.web.server import create_app
|
||||||
|
|
||||||
def client_factory() -> "httpx.AsyncClient":
|
def client_factory() -> httpx.AsyncClient:
|
||||||
return httpx.AsyncClient(
|
return httpx.AsyncClient(
|
||||||
base_url=server_url,
|
base_url=server_url,
|
||||||
headers={
|
headers={
|
||||||
@@ -121,6 +127,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
affect_read_url=affect_read_url,
|
affect_read_url=affect_read_url,
|
||||||
memory_read_url=memory_read_url,
|
memory_read_url=memory_read_url,
|
||||||
admin_key=admin_key,
|
admin_key=admin_key,
|
||||||
|
tts_url=tts_url,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Boot banner to stderr (so stdout stays clean for piping).
|
# Boot banner to stderr (so stdout stays clean for piping).
|
||||||
|
|||||||
@@ -64,6 +64,13 @@ from ratatoskr.sse_client import (
|
|||||||
SseConnectionDropped,
|
SseConnectionDropped,
|
||||||
TurnIdFlip,
|
TurnIdFlip,
|
||||||
)
|
)
|
||||||
|
from ratatoskr.tts import (
|
||||||
|
ZONOS_TTS_URL,
|
||||||
|
PadState,
|
||||||
|
TtsUnavailable,
|
||||||
|
pad_to_dials,
|
||||||
|
tts_synthesize,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _wt_client(
|
def _wt_client(
|
||||||
@@ -529,6 +536,48 @@ async def _memory_chunks_endpoint(request: Request) -> JSONResponse:
|
|||||||
return JSONResponse(r.json(), status_code=r.status_code)
|
return JSONResponse(r.json(), status_code=r.status_code)
|
||||||
|
|
||||||
|
|
||||||
|
# Per-character voice map (DEC-8): interview characters resolve to their registered
|
||||||
|
# Zonos voice; everything else falls to the gateway default. Case-folded gateway-side.
|
||||||
|
_TTS_VOICE_MAP = {"ratatoskr:donut": "donut"}
|
||||||
|
_TTS_DEFAULT_VOICE = "Cora"
|
||||||
|
|
||||||
|
|
||||||
|
async def _tts_endpoint(request: Request) -> Response:
|
||||||
|
"""POST /api/tts {text, agent_id?, pad?} → audio/wav (FN tts_endpoint, slice 2).
|
||||||
|
|
||||||
|
Server-side proxy to the Zonos gateway (DEC-4 / INV-TTS-1: the gateway host
|
||||||
|
never reaches the browser). Voice resolves per-character (DEC-8); emotion dials
|
||||||
|
map the browser-sent live PAD (DEC-7 — the affect the persona pane already shows).
|
||||||
|
Serialized one-synth-at-a-time (DEC-5 / INV-TTS-3 — the gateway shares one 3090).
|
||||||
|
A gateway failure / non-wav body degrades to 503 (INV-TTS-4: the client skips
|
||||||
|
playback; the turn/transcript is unaffected)."""
|
||||||
|
try:
|
||||||
|
body = await request.json()
|
||||||
|
except (json.JSONDecodeError, ValueError, TypeError):
|
||||||
|
body = None
|
||||||
|
text = body.get("text") if isinstance(body, dict) else None
|
||||||
|
if not text or not isinstance(text, str):
|
||||||
|
return JSONResponse({"error_code": "missing_text"}, status_code=400)
|
||||||
|
agent_id = body.get("agent_id") if isinstance(body, dict) else None
|
||||||
|
voice = _TTS_VOICE_MAP.get(agent_id, _TTS_DEFAULT_VOICE)
|
||||||
|
pad_obj = body.get("pad") if isinstance(body, dict) else None
|
||||||
|
dials = pad_to_dials(PadState.from_obj(pad_obj))
|
||||||
|
|
||||||
|
tts_url = request.app.state.tts_url
|
||||||
|
lock = request.app.state.tts_lock
|
||||||
|
try:
|
||||||
|
# DEC-5: serialize — a new turn's synth waits on any in-flight one (the
|
||||||
|
# client also aborts the prior request, cancelling the server task).
|
||||||
|
async with lock:
|
||||||
|
async with httpx.AsyncClient(timeout=httpx.Timeout(120.0)) as client:
|
||||||
|
wav = await tts_synthesize(
|
||||||
|
text, voice=voice, dials=dials, client=client, url=tts_url
|
||||||
|
)
|
||||||
|
except TtsUnavailable:
|
||||||
|
return JSONResponse({"error_code": "tts_unavailable"}, status_code=503)
|
||||||
|
return Response(wav, media_type="audio/wav")
|
||||||
|
|
||||||
|
|
||||||
async def _session_tools_endpoint(request: Request) -> JSONResponse:
|
async def _session_tools_endpoint(request: Request) -> JSONResponse:
|
||||||
"""GET /api/sessions/{session_id}/tools → owner-scoped tool inventory (spec #183).
|
"""GET /api/sessions/{session_id}/tools → owner-scoped tool inventory (spec #183).
|
||||||
|
|
||||||
@@ -673,6 +722,7 @@ def create_app(
|
|||||||
affect_read_url: str | None = None,
|
affect_read_url: str | None = None,
|
||||||
memory_read_url: str | None = None,
|
memory_read_url: str | None = None,
|
||||||
admin_key: str | None = None,
|
admin_key: str | None = None,
|
||||||
|
tts_url: str | None = None,
|
||||||
) -> Starlette:
|
) -> Starlette:
|
||||||
"""Construct the Starlette app — wire routes + state per FN create_app.
|
"""Construct the Starlette app — wire routes + state per FN create_app.
|
||||||
|
|
||||||
@@ -739,6 +789,7 @@ def create_app(
|
|||||||
Route("/api/sessions/{session_id}/messages", _session_messages_endpoint),
|
Route("/api/sessions/{session_id}/messages", _session_messages_endpoint),
|
||||||
Route("/api/sessions/{session_id}/bifrost", _session_bifrost_endpoint),
|
Route("/api/sessions/{session_id}/bifrost", _session_bifrost_endpoint),
|
||||||
Route("/api/admin/events", _admin_events_endpoint),
|
Route("/api/admin/events", _admin_events_endpoint),
|
||||||
|
Route("/api/tts", _tts_endpoint, methods=["POST"]),
|
||||||
Route("/api/turns/{session_id}", _submit_turn_endpoint, methods=["POST"]),
|
Route("/api/turns/{session_id}", _submit_turn_endpoint, methods=["POST"]),
|
||||||
Route("/api/turns/{session_id}/stream", _stream_turn_endpoint),
|
Route("/api/turns/{session_id}/stream", _stream_turn_endpoint),
|
||||||
Route("/api/turns/{session_id}/cancel", _cancel_turn_endpoint, methods=["POST"]),
|
Route("/api/turns/{session_id}/cancel", _cancel_turn_endpoint, methods=["POST"]),
|
||||||
@@ -762,6 +813,12 @@ def create_app(
|
|||||||
# SERVER-HELD (RATATOSKR_ADMIN_API_KEY) and never reaches the browser — the
|
# SERVER-HELD (RATATOSKR_ADMIN_API_KEY) and never reaches the browser — the
|
||||||
# server proxies admin-scoped reads and forwards only the session-filtered result.
|
# server proxies admin-scoped reads and forwards only the session-filtered result.
|
||||||
app.state.admin_key = admin_key
|
app.state.admin_key = admin_key
|
||||||
|
# Auto-TTS (slice 2): the Zonos gateway URL is SERVER-HELD config — the host
|
||||||
|
# never reaches the browser (DEC-4 / INV-TTS-1). Defaults to the direct gateway
|
||||||
|
# (DEC-1); overridable via RATATOSKR_TTS_URL (the swap seam). The lock serializes
|
||||||
|
# one synth at a time so concurrent turns don't contend the shared 3090 (DEC-5).
|
||||||
|
app.state.tts_url = tts_url or ZONOS_TTS_URL
|
||||||
|
app.state.tts_lock = asyncio.Lock()
|
||||||
# INV-002: turn registry is in-process memory, keyed (session_id, turn_id)
|
# INV-002: turn registry is in-process memory, keyed (session_id, turn_id)
|
||||||
app.state.turn_registry = {}
|
app.state.turn_registry = {}
|
||||||
return app
|
return app
|
||||||
|
|||||||
@@ -226,6 +226,12 @@ kbd {
|
|||||||
#theme-toggle .i-moon { display: none; }
|
#theme-toggle .i-moon { display: none; }
|
||||||
body[data-theme="light"] #theme-toggle .i-sun { display: none; }
|
body[data-theme="light"] #theme-toggle .i-sun { display: none; }
|
||||||
body[data-theme="light"] #theme-toggle .i-moon { display: inline; }
|
body[data-theme="light"] #theme-toggle .i-moon { display: inline; }
|
||||||
|
#tts-toggle { width: 30px; height: 30px; display: flex; align-items: center; justify-content: center; background: transparent; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); color: var(--fg-muted); cursor: pointer; flex: 0 0 auto; }
|
||||||
|
#tts-toggle:hover { background: var(--bg-2); color: var(--fg-0); border-color: var(--border-default); }
|
||||||
|
#tts-toggle.on { color: var(--aus-bright-cyan); border-color: var(--border-default); }
|
||||||
|
#tts-toggle .i-wave { display: none; }
|
||||||
|
#tts-toggle.on .i-wave { display: inline; }
|
||||||
|
#tts-toggle.on .i-mute { display: none; }
|
||||||
|
|
||||||
#conn {
|
#conn {
|
||||||
flex: 0 0 auto; margin: 0 14px 14px; padding: 9px 12px;
|
flex: 0 0 auto; margin: 0 14px 14px; padding: 9px 12px;
|
||||||
@@ -573,6 +579,10 @@ body.cot-hidden #cot-toggle { border-color: rgba(66,220,209,0.55); color: var(--
|
|||||||
<div class="brand-name">ratatoskr</div>
|
<div class="brand-name">ratatoskr</div>
|
||||||
<div class="brand-tag">wire monitor</div>
|
<div class="brand-tag">wire monitor</div>
|
||||||
</div>
|
</div>
|
||||||
|
<button id="tts-toggle" title="Voice off — click to toggle auto-TTS">
|
||||||
|
<svg class="i-wave" width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.75" stroke-linecap="round" stroke-linejoin="round"><path d="M11 5 6 9H2v6h4l5 4V5z"></path><path d="M15.5 8.5a5 5 0 0 1 0 7M19 5a9 9 0 0 1 0 14"></path></svg>
|
||||||
|
<svg class="i-mute" width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.75" stroke-linecap="round" stroke-linejoin="round"><path d="M11 5 6 9H2v6h4l5 4V5z"></path><path d="M22 9l-6 6M16 9l6 6"></path></svg>
|
||||||
|
</button>
|
||||||
<button id="theme-toggle" title="Toggle light / dark">
|
<button id="theme-toggle" title="Toggle light / dark">
|
||||||
<svg class="i-sun" width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.75" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="4"></circle><path d="M12 2v2M12 20v2M4.9 4.9l1.4 1.4M17.7 17.7l1.4 1.4M2 12h2M20 12h2M4.9 19.1l1.4-1.4M17.7 6.3l1.4-1.4"></path></svg>
|
<svg class="i-sun" width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.75" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="4"></circle><path d="M12 2v2M12 20v2M4.9 4.9l1.4 1.4M17.7 17.7l1.4 1.4M2 12h2M20 12h2M4.9 19.1l1.4-1.4M17.7 6.3l1.4-1.4"></path></svg>
|
||||||
<svg class="i-moon" width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.75" stroke-linecap="round" stroke-linejoin="round"><path d="M21 12.8A9 9 0 1 1 11.2 3a7 7 0 0 0 9.8 9.8z"></path></svg>
|
<svg class="i-moon" width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.75" stroke-linecap="round" stroke-linejoin="round"><path d="M21 12.8A9 9 0 1 1 11.2 3a7 7 0 0 0 9.8 9.8z"></path></svg>
|
||||||
@@ -1698,6 +1708,7 @@ async function submitPrompt() {
|
|||||||
const input = $("prompt-input");
|
const input = $("prompt-input");
|
||||||
const content = input.value.trim();
|
const content = input.value.trim();
|
||||||
if (!content || !state.sessionId || state.turnId) return;
|
if (!content || !state.sessionId || state.turnId) return;
|
||||||
|
cancelTts(); // INV-TTS-3: a new turn cancels prior voice
|
||||||
input.value = ""; input.style.height = "20px";
|
input.value = ""; input.style.height = "20px";
|
||||||
|
|
||||||
let r;
|
let r;
|
||||||
@@ -1811,6 +1822,11 @@ async function submitPrompt() {
|
|||||||
tickerAdd(cls === "done" ? "ok" : cls === "error" ? "err" : "turn",
|
tickerAdd(cls === "done" ? "ok" : cls === "error" ? "err" : "turn",
|
||||||
cls === "done" ? "turn.completed" : cls === "error" ? "turn.failed" : "turn.cancelled",
|
cls === "done" ? "turn.completed" : cls === "error" ? "turn.failed" : "turn.cancelled",
|
||||||
cls === "done" ? elapsed + "s" : meta);
|
cls === "done" ? elapsed + "s" : meta);
|
||||||
|
// Auto-TTS (slice 2, DEC-7): speak the completed response, emotion-modulated by the
|
||||||
|
// live PAD the persona pane shows. Only on `done`; opt-in; failures are non-blocking.
|
||||||
|
if (cls === "done" && ttsEnabled()) {
|
||||||
|
speakOnDone(LIVE.resp, state.agentId, state.lastSnap && state.lastSnap.pad);
|
||||||
|
}
|
||||||
es.close();
|
es.close();
|
||||||
state.eventSource = null; state.turnId = null; state.curTurnEl = null;
|
state.eventSource = null; state.turnId = null; state.curTurnEl = null;
|
||||||
$("composer").classList.remove("streaming");
|
$("composer").classList.remove("streaming");
|
||||||
@@ -1977,6 +1993,65 @@ async function cancelTurn() {
|
|||||||
});
|
});
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
// ---- auto-TTS (slice 2): voiced, affect-modulated playback on turn `done` -----
|
||||||
|
// Server-proxied to the Zonos gateway (/api/tts). Opt-in (INV-TTS-2), one clip at a
|
||||||
|
// time (INV-TTS-3: a new turn or a superseding synth cancels the prior fetch + audio),
|
||||||
|
// and non-blocking (INV-TTS-4: any failure logs to the ticker + skips — never the turn).
|
||||||
|
let _ttsAbort = null;
|
||||||
|
function ttsEnabled() {
|
||||||
|
try { return localStorage.getItem("ratatoskr-tts") === "1"; } catch (_) { return false; }
|
||||||
|
}
|
||||||
|
function cancelTts() {
|
||||||
|
if (_ttsAbort) { try { _ttsAbort.abort(); } catch (_) {} _ttsAbort = null; }
|
||||||
|
const a = $("tts-audio");
|
||||||
|
if (a) { try { a.pause(); a.removeAttribute("src"); a.load(); } catch (_) {} }
|
||||||
|
}
|
||||||
|
async function speakOnDone(text, agentId, pad) {
|
||||||
|
const clip = (text || "").trim();
|
||||||
|
if (!clip) return;
|
||||||
|
cancelTts(); // INV-TTS-3: stop any prior synth/playback
|
||||||
|
const ctrl = new AbortController();
|
||||||
|
_ttsAbort = ctrl;
|
||||||
|
let resp;
|
||||||
|
try {
|
||||||
|
resp = await fetch("/api/tts", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({ text: clip, agent_id: agentId || null, pad: pad || null }),
|
||||||
|
signal: ctrl.signal,
|
||||||
|
});
|
||||||
|
} catch (_) { return; } // aborted / network → silent skip (INV-TTS-4)
|
||||||
|
if (!resp.ok) { tickerAdd("err", "tts", "unavailable " + resp.status); return; }
|
||||||
|
let url;
|
||||||
|
try { url = URL.createObjectURL(await resp.blob()); } catch (_) { return; }
|
||||||
|
if (_ttsAbort !== ctrl) { URL.revokeObjectURL(url); return; } // superseded mid-fetch
|
||||||
|
const a = $("tts-audio");
|
||||||
|
if (!a) { URL.revokeObjectURL(url); return; }
|
||||||
|
a.src = url;
|
||||||
|
a.onended = a.onerror = () => { URL.revokeObjectURL(url); };
|
||||||
|
// Playback can be refused by the browser autoplay policy until the page has an
|
||||||
|
// activation; the operator's toggle+prompt gesture generally satisfies it, and the
|
||||||
|
// catch keeps a refusal non-fatal (INV-TTS-4).
|
||||||
|
try { await a.play(); tickerAdd("ok", "tts", "▶ voiced"); }
|
||||||
|
catch (_) { URL.revokeObjectURL(url); tickerAdd("err", "tts", "playback blocked"); }
|
||||||
|
}
|
||||||
|
// ---- 🔊 toggle (mirrors theme / cot-toggle; default OFF, persisted) ----
|
||||||
|
(function () {
|
||||||
|
let on = false;
|
||||||
|
try { on = localStorage.getItem("ratatoskr-tts") === "1"; } catch (_) {}
|
||||||
|
const btn = $("tts-toggle");
|
||||||
|
const label = (v) => "Voice " + (v ? "on" : "off") + " — click to toggle auto-TTS";
|
||||||
|
btn.classList.toggle("on", on);
|
||||||
|
btn.title = label(on);
|
||||||
|
btn.addEventListener("click", () => {
|
||||||
|
const next = !btn.classList.contains("on");
|
||||||
|
btn.classList.toggle("on", next);
|
||||||
|
btn.title = label(next);
|
||||||
|
try { localStorage.setItem("ratatoskr-tts", next ? "1" : "0"); } catch (_) {}
|
||||||
|
if (!next) cancelTts(); // turning off stops in-flight playback
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
|
||||||
// ---- collapsible debug panes (independent left / right; state persists) ----
|
// ---- collapsible debug panes (independent left / right; state persists) ----
|
||||||
(function () {
|
(function () {
|
||||||
const cols = $("cols");
|
const cols = $("cols");
|
||||||
@@ -2054,5 +2129,6 @@ loadAgents();
|
|||||||
loadVersion();
|
loadVersion();
|
||||||
loadPersonaCanon();
|
loadPersonaCanon();
|
||||||
</script>
|
</script>
|
||||||
|
<audio id="tts-audio" hidden preload="none"></audio>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -0,0 +1,168 @@
|
|||||||
|
"""Tests for ratatoskr.tts per docs/contracts/donut_voiced_interview.contract.md
|
||||||
|
(slice 2 — auto-TTS). Two units:
|
||||||
|
|
||||||
|
- pad_to_dials / PadState / EmotionDials — pure, total, degrade-not-crash.
|
||||||
|
- tts_synthesize — the Zonos-gateway client (respx-mocked; no live network).
|
||||||
|
|
||||||
|
DEC-3: response_format is ALWAYS "wav". DEC-7: live PAD → emotion dials.
|
||||||
|
INV-TTS-4: a gateway failure / non-wav body → TtsUnavailable (caller degrades).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import math
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
import respx
|
||||||
|
|
||||||
|
from ratatoskr.tts import (
|
||||||
|
EmotionDials,
|
||||||
|
PadState,
|
||||||
|
TtsUnavailable,
|
||||||
|
pad_to_dials,
|
||||||
|
tts_synthesize,
|
||||||
|
)
|
||||||
|
|
||||||
|
# A minimally-valid wav body: the DEC-3 guard only checks the RIFF magic.
|
||||||
|
_WAV = b"RIFF" + b"\x00" * 40
|
||||||
|
|
||||||
|
|
||||||
|
class TestPadState:
|
||||||
|
def test_from_obj_valid_mapping(self) -> None:
|
||||||
|
pad = PadState.from_obj({"pleasure": 0.5, "arousal": -0.2, "dominance": 0.1})
|
||||||
|
assert pad == PadState(pleasure=0.5, arousal=-0.2, dominance=0.1)
|
||||||
|
|
||||||
|
def test_from_obj_dominance_optional(self) -> None:
|
||||||
|
pad = PadState.from_obj({"pleasure": 0.5, "arousal": -0.2})
|
||||||
|
assert pad is not None and pad.dominance == 0.0
|
||||||
|
|
||||||
|
def test_from_obj_none_is_none(self) -> None:
|
||||||
|
assert PadState.from_obj(None) is None
|
||||||
|
|
||||||
|
def test_from_obj_non_mapping_is_none(self) -> None:
|
||||||
|
assert PadState.from_obj("not a mapping") is None
|
||||||
|
assert PadState.from_obj([0.1, 0.2]) is None
|
||||||
|
|
||||||
|
def test_from_obj_missing_key_is_none(self) -> None:
|
||||||
|
assert PadState.from_obj({"pleasure": 0.5}) is None # no arousal
|
||||||
|
|
||||||
|
def test_from_obj_non_numeric_is_none(self) -> None:
|
||||||
|
assert PadState.from_obj({"pleasure": "hot", "arousal": 0.1}) is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestPadToDials:
|
||||||
|
def test_none_pad_is_neutral_disabled(self) -> None:
|
||||||
|
d = pad_to_dials(None)
|
||||||
|
assert d.emotion_enabled is False
|
||||||
|
assert d.to_body() == {} # a neutral read sends no emotion params
|
||||||
|
|
||||||
|
def test_maps_pleasure_and_arousal(self) -> None:
|
||||||
|
d = pad_to_dials(PadState(pleasure=0.4, arousal=0.6))
|
||||||
|
assert d.emotion_enabled is True
|
||||||
|
assert d.emotion_valence == pytest.approx(0.4)
|
||||||
|
assert d.emotion_arousal == pytest.approx(0.6)
|
||||||
|
|
||||||
|
def test_clamps_out_of_range(self) -> None:
|
||||||
|
d = pad_to_dials(PadState(pleasure=5.0, arousal=-9.0))
|
||||||
|
assert d.emotion_valence == 1.0
|
||||||
|
assert d.emotion_arousal == -1.0
|
||||||
|
|
||||||
|
def test_nan_degrades_to_zero_never_raises(self) -> None:
|
||||||
|
# INV FN pad_to_dials: total over ANY input (incl. NaN) → valid dials.
|
||||||
|
d = pad_to_dials(PadState(pleasure=math.nan, arousal=math.inf))
|
||||||
|
assert d.emotion_valence == 0.0
|
||||||
|
assert d.emotion_arousal == 1.0 # +inf clamps to the ceiling
|
||||||
|
assert not math.isnan(d.emotion_valence)
|
||||||
|
|
||||||
|
|
||||||
|
class TestEmotionDialsToBody:
|
||||||
|
def test_disabled_emits_no_params(self) -> None:
|
||||||
|
assert EmotionDials(emotion_enabled=False).to_body() == {}
|
||||||
|
|
||||||
|
def test_enabled_emits_valence_arousal_strength(self) -> None:
|
||||||
|
body = EmotionDials(
|
||||||
|
emotion_enabled=True, emotion_valence=0.3, emotion_arousal=-0.1
|
||||||
|
).to_body()
|
||||||
|
assert body["emotion_enabled"] is True
|
||||||
|
assert body["emotion_valence"] == 0.3
|
||||||
|
assert body["emotion_arousal"] == -0.1
|
||||||
|
assert "emotion_strength" in body
|
||||||
|
|
||||||
|
|
||||||
|
class TestTtsSynthesize:
|
||||||
|
@respx.mock
|
||||||
|
async def test_happy_posts_wav_request_and_returns_bytes(self) -> None:
|
||||||
|
route = respx.post("http://tts.example/v1/audio/speech").mock(
|
||||||
|
return_value=httpx.Response(200, content=_WAV)
|
||||||
|
)
|
||||||
|
async with httpx.AsyncClient() as client:
|
||||||
|
out = await tts_synthesize(
|
||||||
|
"hello there",
|
||||||
|
voice="donut",
|
||||||
|
dials=pad_to_dials(PadState(pleasure=0.5, arousal=0.2)),
|
||||||
|
client=client,
|
||||||
|
url="http://tts.example/v1/audio/speech",
|
||||||
|
)
|
||||||
|
assert out == _WAV
|
||||||
|
sent = route.calls.last.request
|
||||||
|
import json as _json
|
||||||
|
|
||||||
|
body = _json.loads(sent.content)
|
||||||
|
assert body["input"] == "hello there"
|
||||||
|
assert body["voice"] == "donut"
|
||||||
|
assert body["response_format"] == "wav" # DEC-3 — ALWAYS wav
|
||||||
|
assert body["emotion_valence"] == pytest.approx(0.5)
|
||||||
|
|
||||||
|
@respx.mock
|
||||||
|
async def test_neutral_dials_send_no_emotion_params(self) -> None:
|
||||||
|
route = respx.post("http://tts.example/v1/audio/speech").mock(
|
||||||
|
return_value=httpx.Response(200, content=_WAV)
|
||||||
|
)
|
||||||
|
async with httpx.AsyncClient() as client:
|
||||||
|
await tts_synthesize(
|
||||||
|
"hi", voice="Cora", dials=pad_to_dials(None), client=client,
|
||||||
|
url="http://tts.example/v1/audio/speech",
|
||||||
|
)
|
||||||
|
import json as _json
|
||||||
|
|
||||||
|
body = _json.loads(route.calls.last.request.content)
|
||||||
|
assert "emotion_valence" not in body
|
||||||
|
assert body["response_format"] == "wav"
|
||||||
|
|
||||||
|
@respx.mock
|
||||||
|
async def test_non_200_raises_tts_unavailable(self) -> None:
|
||||||
|
respx.post("http://tts.example/v1/audio/speech").mock(
|
||||||
|
return_value=httpx.Response(500, content=b"boom")
|
||||||
|
)
|
||||||
|
async with httpx.AsyncClient() as client:
|
||||||
|
with pytest.raises(TtsUnavailable) as exc:
|
||||||
|
await tts_synthesize(
|
||||||
|
"hi", voice="Cora", dials=pad_to_dials(None), client=client,
|
||||||
|
url="http://tts.example/v1/audio/speech",
|
||||||
|
)
|
||||||
|
assert exc.value.status == 500
|
||||||
|
|
||||||
|
@respx.mock
|
||||||
|
async def test_transport_error_raises_tts_unavailable(self) -> None:
|
||||||
|
respx.post("http://tts.example/v1/audio/speech").mock(
|
||||||
|
side_effect=httpx.ConnectError("refused")
|
||||||
|
)
|
||||||
|
async with httpx.AsyncClient() as client:
|
||||||
|
with pytest.raises(TtsUnavailable):
|
||||||
|
await tts_synthesize(
|
||||||
|
"hi", voice="Cora", dials=pad_to_dials(None), client=client,
|
||||||
|
url="http://tts.example/v1/audio/speech",
|
||||||
|
)
|
||||||
|
|
||||||
|
@respx.mock
|
||||||
|
async def test_non_wav_body_raises_tts_unavailable(self) -> None:
|
||||||
|
# DEC-3 guard: a 200 with a non-RIFF body (an error page, or the
|
||||||
|
# mislabeled-PCM mp3/opus trap) is unavailable, not played.
|
||||||
|
respx.post("http://tts.example/v1/audio/speech").mock(
|
||||||
|
return_value=httpx.Response(200, content=b"<html>error</html>")
|
||||||
|
)
|
||||||
|
async with httpx.AsyncClient() as client:
|
||||||
|
with pytest.raises(TtsUnavailable):
|
||||||
|
await tts_synthesize(
|
||||||
|
"hi", voice="Cora", dials=pad_to_dials(None), client=client,
|
||||||
|
url="http://tts.example/v1/audio/speech",
|
||||||
|
)
|
||||||
@@ -6,6 +6,8 @@ client_factory callable; tests pass a factory returning a respx-mocked
|
|||||||
AsyncClient.
|
AsyncClient.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
import respx
|
import respx
|
||||||
@@ -1443,3 +1445,84 @@ class TestMemoryChunksEndpoint:
|
|||||||
)
|
)
|
||||||
resp = TestClient(app).get("/api/memory/chunks?agent_id=a")
|
resp = TestClient(app).get("/api/memory/chunks?agent_id=a")
|
||||||
assert resp.status_code == 400
|
assert resp.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
class TestTtsEndpoint:
|
||||||
|
"""tts_endpoint FN — POST /api/tts → audio/wav via the Zonos gateway proxy
|
||||||
|
(slice 2, docs/contracts/donut_voiced_interview.contract.md). The gateway host
|
||||||
|
never reaches the browser (INV-TTS-1/DEC-4); voice resolves per-character
|
||||||
|
(DEC-8); emotion dials map the browser-sent live PAD (DEC-7); a gateway failure
|
||||||
|
degrades to 503 (INV-TTS-4)."""
|
||||||
|
|
||||||
|
_WAV = b"RIFF" + b"\x00" * 40
|
||||||
|
_TTS = "http://tts.example/v1/audio/speech"
|
||||||
|
|
||||||
|
@respx.mock
|
||||||
|
def test_happy_returns_wav_and_resolves_donut_voice_and_pad(self) -> None:
|
||||||
|
from ratatoskr.web.server import create_app
|
||||||
|
|
||||||
|
route = respx.post(self._TTS).mock(
|
||||||
|
return_value=httpx.Response(200, content=self._WAV)
|
||||||
|
)
|
||||||
|
app = create_app(_mock_client_factory(), tts_url=self._TTS)
|
||||||
|
resp = TestClient(app).post(
|
||||||
|
"/api/tts",
|
||||||
|
json={
|
||||||
|
"text": "Carl looks intimidating, but he's a softie.",
|
||||||
|
"agent_id": "ratatoskr:donut",
|
||||||
|
"pad": {"pleasure": 0.6, "arousal": 0.3, "dominance": 0.1},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert resp.headers["content-type"] == "audio/wav"
|
||||||
|
assert resp.content == self._WAV
|
||||||
|
# DEC-8 voice map + DEC-7 affect dials rode the gateway request.
|
||||||
|
body = json.loads(route.calls.last.request.content)
|
||||||
|
assert body["voice"] == "donut"
|
||||||
|
assert body["response_format"] == "wav"
|
||||||
|
assert body["emotion_valence"] == pytest.approx(0.6)
|
||||||
|
|
||||||
|
@respx.mock
|
||||||
|
def test_unmapped_agent_uses_default_voice_and_no_pad_no_emotion(self) -> None:
|
||||||
|
from ratatoskr.web.server import create_app
|
||||||
|
|
||||||
|
route = respx.post(self._TTS).mock(
|
||||||
|
return_value=httpx.Response(200, content=self._WAV)
|
||||||
|
)
|
||||||
|
app = create_app(_mock_client_factory(), tts_url=self._TTS)
|
||||||
|
resp = TestClient(app).post(
|
||||||
|
"/api/tts", json={"text": "hello", "agent_id": "mimir"}
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = json.loads(route.calls.last.request.content)
|
||||||
|
assert body["voice"] == "Cora" # gateway default (DEC-8)
|
||||||
|
assert "emotion_valence" not in body # no PAD → neutral read (DEC-7)
|
||||||
|
|
||||||
|
def test_missing_text_returns_400(self) -> None:
|
||||||
|
from ratatoskr.web.server import create_app
|
||||||
|
|
||||||
|
app = create_app(_mock_client_factory(), tts_url=self._TTS)
|
||||||
|
resp = TestClient(app).post("/api/tts", json={"agent_id": "ratatoskr:donut"})
|
||||||
|
assert resp.status_code == 400
|
||||||
|
assert resp.json()["error_code"] == "missing_text"
|
||||||
|
|
||||||
|
@respx.mock
|
||||||
|
def test_gateway_500_degrades_to_503(self) -> None:
|
||||||
|
from ratatoskr.web.server import create_app
|
||||||
|
|
||||||
|
respx.post(self._TTS).mock(return_value=httpx.Response(500, content=b"boom"))
|
||||||
|
app = create_app(_mock_client_factory(), tts_url=self._TTS)
|
||||||
|
resp = TestClient(app).post("/api/tts", json={"text": "hi"})
|
||||||
|
assert resp.status_code == 503
|
||||||
|
assert resp.json()["error_code"] == "tts_unavailable"
|
||||||
|
|
||||||
|
@respx.mock
|
||||||
|
def test_non_wav_body_degrades_to_503(self) -> None:
|
||||||
|
from ratatoskr.web.server import create_app
|
||||||
|
|
||||||
|
respx.post(self._TTS).mock(
|
||||||
|
return_value=httpx.Response(200, content=b"<html>nope</html>")
|
||||||
|
)
|
||||||
|
app = create_app(_mock_client_factory(), tts_url=self._TTS)
|
||||||
|
resp = TestClient(app).post("/api/tts", json={"text": "hi"})
|
||||||
|
assert resp.status_code == 503
|
||||||
|
|||||||
Reference in New Issue
Block a user