vh/ratatoskr
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stream Donut TTS play-as-it-arrives + autoplay unlock (supersedes buffered)

Browse Source 
Operator: play-as-it-arrives, don't wait for the whole clip. infra-ops confirmed the
Zonos gateway ALREADY streams (chunked int16 WAV, TTFB ~0.44s vs ~7s total; placeholder
0xFFFFFFFF sizes are DESIGNED for progressive <audio src>). The buffering was entirely
in our proxy, and the _finalize_wav_header rewrite (6c3c08b) FORCED it — computing the
real sizes needs the whole clip.

The fix — pipe the chunks straight through:
- tts.py: buffered tts_synthesize + _finalize_wav_header REMOVED; tts_stream (an async
  generator over the gateway's chunked response) + gateway_body added. Never buffer,
  never rewrite the placeholder header.
- server.py: /api/tts is now GET (so a browser <audio src> plays it progressively) →
  a chunked StreamingResponse piping the gateway; peeks the first chunk so a bad gateway
  OPEN still returns 503; the serialize lock is held across the stream and released on
  completion/abort; PAD rides p/a query floats.
- index.html: speakOnDone sets <audio src="/api/tts?..."> (streaming) instead of
  fetch->blob; dropped the blob machinery. AUTOPLAY UNLOCK: _unlockTtsAudio() plays a
  silent WAV within the toggle/submit gesture so the delayed play() isn't blocked — the
  actual cause of "no audio" (play() fires ~15s after the keypress, past the browser's
  transient-activation window).

Live-verified: GET /api/tts is transfer-encoding: chunked, TTFB 0.46s. Playwright with
--autoplay-policy=document-user-activation-required: the streaming <audio src> plays
progressively (currentTime advances, no decode error, no MSE fallback needed) 6.5s after
the gesture — proving the unlock's persistent element flag. 521 green.

DEC-2 amended (streaming supersedes "no streaming"); FN tts_stream / tts_endpoint updated.
This commit is contained in:
Vuong Hoang
2026-08-01 23:59:35 -07:00
parent 608e9a54fd
commit 7856ec5438
6 changed files with 272 additions and 359 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_tts.py
+54 -133
View File
@@ -1,17 +1,11 @@
"""Tests for ratatoskr.tts per docs/contracts/donut_voiced_interview.contract.md
(slice 2 — auto-TTS). Two units:
"""Tests for ratatoskr.tts — the STREAMING Zonos-gateway client + PAD→dial mapping.
- pad_to_dials / PadState / EmotionDials — pure, total, degrade-not-crash.
- tts_synthesize — the Zonos-gateway client (respx-mocked; no live network).
DEC-3: response_format is ALWAYS "wav". DEC-7: live PAD → emotion dials.
INV-TTS-4: a gateway failure / non-wav body → TtsUnavailable (caller degrades).
tts_stream proxies the gateway's chunked response verbatim (no buffering, no header
rewrite — the placeholder-size streaming WAV is meant to be played progressively).
pad_to_dials / PadState / EmotionDials are pure + total.
"""
import io
import math
import struct
import wave
import httpx
import pytest
@@ -21,24 +15,25 @@ from ratatoskr.tts import (
EmotionDials,
PadState,
TtsUnavailable,
gateway_body,
pad_to_dials,
tts_synthesize,
tts_stream,
)
_URL = "http://tts.example/v1/audio/speech"
# The gateway's streaming WAV bytes (placeholder 0xFFFFFFFF sizes). We pass them through
# untouched, so the content only has to round-trip.
_WAV = (
b"RIFF\xff\xff\xff\xffWAVEfmt \x10\x00\x00\x00" + b"\x00" * 20
+ b"data\xff\xff\xff\xff" + b"\x11\x22" * 64
)
def _wav(*, streaming: bool = False, nsamples: int = 8) -> bytes:
"""A canonical PCM mono/44.1k/16-bit WAV. `streaming=True` mimics the Zonos
gateway's placeholder header (RIFF + data sizes = 0xFFFFFFFF)."""
samples = b"\x11\x22" * nsamples
fmt = struct.pack("<HHIIHH", 1, 1, 44100, 88200, 2, 16)
body = b"WAVE" + b"fmt " + struct.pack("<I", 16) + fmt + b"data"
data_size = 0xFFFFFFFF if streaming else len(samples)
riff_size = 0xFFFFFFFF if streaming else (len(body) + 4 + len(samples))
return b"RIFF" + struct.pack("<I", riff_size) + body + struct.pack("<I", data_size) + samples
# A proper finite WAV — the header-finalize step is idempotent on it (sizes already real).
_WAV = _wav()
async def _drain(gen) -> bytes:
out = b""
async for chunk in gen:
out += chunk
return out
class TestPadState:
@@ -64,8 +59,6 @@ class TestPadState:
assert PadState.from_obj({"pleasure": "hot", "arousal": 0.1}) is None
def test_from_obj_huge_int_overflow_is_none(self) -> None:
# float() of a 400-digit JSON integer raises OverflowError (an ArithmeticError,
# not a ValueError); from_obj must still degrade to None, not propagate a 500.
huge = int("9" * 400)
assert PadState.from_obj({"pleasure": huge, "arousal": 0}) is None
@@ -74,7 +67,7 @@ class TestPadToDials:
def test_none_pad_is_neutral_disabled(self) -> None:
d = pad_to_dials(None)
assert d.emotion_enabled is False
assert d.to_body() == {} # a neutral read sends no emotion params
assert d.to_body() == {}
def test_maps_pleasure_and_arousal(self) -> None:
d = pad_to_dials(PadState(pleasure=0.4, arousal=0.6))
@@ -88,19 +81,14 @@ class TestPadToDials:
assert d.emotion_arousal == -1.0
def test_nan_degrades_to_zero_never_raises(self) -> None:
# INV FN pad_to_dials: total over ANY input (incl. NaN) → valid dials.
d = pad_to_dials(PadState(pleasure=math.nan, arousal=math.inf))
assert d.emotion_valence == 0.0
assert d.emotion_arousal == 1.0 # +inf clamps to the ceiling
assert not math.isnan(d.emotion_valence)
assert d.emotion_arousal == 1.0
def test_non_padstate_input_degrades_to_neutral(self) -> None:
# The declared surface is PadState | None, but the function must be total over
# ANY object (a raw dict / str / arbitrary object) → neutral, never raise.
for bad in ({}, "bad", [0.1, 0.2], object(), 42):
d = pad_to_dials(bad)
assert d.emotion_enabled is False
assert d.to_body() == {}
assert d.emotion_enabled is False and d.to_body() == {}
class TestEmotionDialsToBody:
@@ -117,118 +105,51 @@ class TestEmotionDialsToBody:
assert "emotion_strength" in body
class TestTtsSynthesize:
class TestGatewayBody:
def test_always_wav_with_dials(self) -> None:
b = gateway_body("hi", "donut", pad_to_dials(PadState(pleasure=0.5, arousal=0.2)))
assert b["input"] == "hi" and b["voice"] == "donut"
assert b["response_format"] == "wav" # DEC-3 — ALWAYS wav
assert b["emotion_valence"] == pytest.approx(0.5)
def test_neutral_omits_emotion(self) -> None:
b = gateway_body("hi", "Cora", pad_to_dials(None))
assert "emotion_valence" not in b and b["response_format"] == "wav"
class TestTtsStream:
@respx.mock
async def test_happy_posts_wav_request_and_returns_bytes(self) -> None:
route = respx.post("http://tts.example/v1/audio/speech").mock(
return_value=httpx.Response(200, content=_WAV)
)
async def test_streams_chunks_and_posts_wav_body(self) -> None:
route = respx.post(_URL).mock(return_value=httpx.Response(200, content=_WAV))
async with httpx.AsyncClient() as client:
out = await tts_synthesize(
"hello there",
voice="donut",
out = await _drain(tts_stream(
"hello there", voice="donut",
dials=pad_to_dials(PadState(pleasure=0.5, arousal=0.2)),
client=client,
url="http://tts.example/v1/audio/speech",
)
assert out == _WAV
sent = route.calls.last.request
import json as _json
body = _json.loads(sent.content)
assert body["input"] == "hello there"
assert body["voice"] == "donut"
assert body["response_format"] == "wav" # DEC-3 — ALWAYS wav
assert body["emotion_valence"] == pytest.approx(0.5)
@respx.mock
async def test_neutral_dials_send_no_emotion_params(self) -> None:
route = respx.post("http://tts.example/v1/audio/speech").mock(
return_value=httpx.Response(200, content=_WAV)
)
async with httpx.AsyncClient() as client:
await tts_synthesize(
"hi", voice="Cora", dials=pad_to_dials(None), client=client,
url="http://tts.example/v1/audio/speech",
)
client=client, url=_URL,
))
assert out == _WAV # passed through verbatim — no header rewrite
import json as _json
body = _json.loads(route.calls.last.request.content)
assert "emotion_valence" not in body
assert body["voice"] == "donut"
assert body["response_format"] == "wav"
assert body["emotion_valence"] == pytest.approx(0.5)
@respx.mock
async def test_non_200_raises_tts_unavailable(self) -> None:
respx.post("http://tts.example/v1/audio/speech").mock(
return_value=httpx.Response(500, content=b"boom")
)
async def test_non_200_open_raises_before_any_chunk(self) -> None:
respx.post(_URL).mock(return_value=httpx.Response(500, content=b"boom"))
async with httpx.AsyncClient() as client:
with pytest.raises(TtsUnavailable) as exc:
await tts_synthesize(
"hi", voice="Cora", dials=pad_to_dials(None), client=client,
url="http://tts.example/v1/audio/speech",
)
await _drain(tts_stream(
"hi", voice="Cora", dials=pad_to_dials(None), client=client, url=_URL
))
assert exc.value.status == 500
@respx.mock
async def test_transport_error_raises_tts_unavailable(self) -> None:
respx.post("http://tts.example/v1/audio/speech").mock(
side_effect=httpx.ConnectError("refused")
)
async def test_transport_error_raises(self) -> None:
respx.post(_URL).mock(side_effect=httpx.ConnectError("refused"))
async with httpx.AsyncClient() as client:
with pytest.raises(TtsUnavailable):
await tts_synthesize(
"hi", voice="Cora", dials=pad_to_dials(None), client=client,
url="http://tts.example/v1/audio/speech",
)
@respx.mock
async def test_non_wav_body_raises_tts_unavailable(self) -> None:
# DEC-3 guard: a 200 with a non-RIFF body (an error page, or the
# mislabeled-PCM mp3/opus trap) is unavailable, not played.
respx.post("http://tts.example/v1/audio/speech").mock(
return_value=httpx.Response(200, content=b"<html>error</html>")
)
async with httpx.AsyncClient() as client:
with pytest.raises(TtsUnavailable):
await tts_synthesize(
"hi", voice="Cora", dials=pad_to_dials(None), client=client,
url="http://tts.example/v1/audio/speech",
)
@respx.mock
async def test_finalizes_streaming_wav_header_to_real_sizes(self) -> None:
# The Zonos gateway returns a streaming header (RIFF + data sizes = 0xFFFFFFFF);
# the browser <audio> element can't play that. tts_synthesize must rewrite both
# to the real byte counts so the result is a decodable finite WAV.
streaming = _wav(streaming=True)
assert struct.unpack("<I", streaming[4:8])[0] == 0xFFFFFFFF # precondition
respx.post("http://tts.example/v1/audio/speech").mock(
return_value=httpx.Response(200, content=streaming)
)
async with httpx.AsyncClient() as client:
out = await tts_synthesize(
"hi", voice="Cora", dials=pad_to_dials(None), client=client,
url="http://tts.example/v1/audio/speech",
)
# RIFF size + data size are now real, not 0xFFFFFFFF.
assert struct.unpack("<I", out[4:8])[0] == len(out) - 8
dpos = out.find(b"data", 12)
assert struct.unpack("<I", out[dpos + 4:dpos + 8])[0] == len(out) - (dpos + 8)
# …and Python's wave module (a real decoder) can now open it as a finite clip.
with wave.open(io.BytesIO(out)) as w:
assert w.getnchannels() == 1 and w.getsampwidth() == 2 and w.getframerate() == 44100
@respx.mock
async def test_riff_but_not_wave_raises_tts_unavailable(self) -> None:
# A body with the RIFF magic but a non-WAVE form tag (e.g. RIFF/AVI, or a
# truncated/garbage container) is not playable audio — reject it too.
respx.post("http://tts.example/v1/audio/speech").mock(
return_value=httpx.Response(200, content=b"RIFF\x00\x00\x00\x00AVI LIST")
)
async with httpx.AsyncClient() as client:
with pytest.raises(TtsUnavailable):
await tts_synthesize(
"hi", voice="Cora", dials=pad_to_dials(None), client=client,
url="http://tts.example/v1/audio/speech",
)
await _drain(tts_stream(
"hi", voice="Cora", dials=pad_to_dials(None), client=client, url=_URL
))
tests/test_web_server.py
+30 -68
View File
@@ -7,7 +7,6 @@ AsyncClient.
"""
import json
import struct
import httpx
import pytest
@@ -1459,68 +1458,55 @@ class TestMemoryChunksEndpoint:
class TestTtsEndpoint:
"""tts_endpoint FN — POST /api/tts → audio/wav via the Zonos gateway proxy
(slice 2, docs/contracts/donut_voiced_interview.contract.md). The gateway host
never reaches the browser (INV-TTS-1/DEC-4); voice resolves per-character
(DEC-8); emotion dials map the browser-sent live PAD (DEC-7); a gateway failure
degrades to 503 (INV-TTS-4)."""
"""tts_endpoint FN — GET /api/tts → audio/wav STREAMED (chunked) from the Zonos
gateway. Voice per-character (DEC-8), emotion dials from p/a query floats (DEC-7),
the gateway host never reaches the browser (INV-TTS-1), gateway open-failure → 503
(INV-TTS-4). GET so a browser <audio src> plays it progressively."""
# A proper finite PCM WAV (correct RIFF + data sizes) so the header-finalize step is
# idempotent — the endpoint returns these bytes unchanged.
_WAV = (
b"RIFF" + (36 + 16).to_bytes(4, "little") + b"WAVE"
+ b"fmt " + (16).to_bytes(4, "little")
+ struct.pack("<HHIIHH", 1, 1, 44100, 88200, 2, 16)
+ b"data" + (16).to_bytes(4, "little") + b"\x11\x22" * 8
)
# Streaming WAV bytes (placeholder 0xFFFFFFFF sizes) — proxied through verbatim.
_WAV = b"RIFF\xff\xff\xff\xffWAVEdata\xff\xff\xff\xff" + b"\x11\x22" * 64
_TTS = "http://tts.example/v1/audio/speech"
@respx.mock
def test_happy_returns_wav_and_resolves_donut_voice_and_pad(self) -> None:
def test_happy_streams_wav_resolves_donut_voice_and_pad(self) -> None:
from ratatoskr.web.server import create_app
route = respx.post(self._TTS).mock(
return_value=httpx.Response(200, content=self._WAV)
)
app = create_app(_mock_client_factory(), tts_url=self._TTS)
resp = TestClient(app).post(
resp = TestClient(app).get(
"/api/tts",
json={
"text": "Carl looks intimidating, but he's a softie.",
"agent_id": "ratatoskr:donut",
"pad": {"pleasure": 0.6, "arousal": 0.3, "dominance": 0.1},
},
params={"text": "Carl is a softie.", "agent_id": "ratatoskr:donut",
"p": "0.6", "a": "0.3"},
)
assert resp.status_code == 200
assert resp.headers["content-type"] == "audio/wav"
assert resp.content == self._WAV
# DEC-8 voice map + DEC-7 affect dials rode the gateway request.
assert resp.headers["content-type"].startswith("audio/wav")
assert resp.content == self._WAV # streamed through verbatim, no header rewrite
body = json.loads(route.calls.last.request.content)
assert body["voice"] == "donut"
assert body["response_format"] == "wav"
assert body["emotion_valence"] == pytest.approx(0.6)
@respx.mock
def test_unmapped_agent_uses_default_voice_and_no_pad_no_emotion(self) -> None:
def test_unmapped_agent_default_voice_no_pad_no_emotion(self) -> None:
from ratatoskr.web.server import create_app
route = respx.post(self._TTS).mock(
return_value=httpx.Response(200, content=self._WAV)
)
app = create_app(_mock_client_factory(), tts_url=self._TTS)
resp = TestClient(app).post(
"/api/tts", json={"text": "hello", "agent_id": "mimir"}
)
resp = TestClient(app).get("/api/tts", params={"text": "hello", "agent_id": "mimir"})
assert resp.status_code == 200
body = json.loads(route.calls.last.request.content)
assert body["voice"] == "Cora" # gateway default (DEC-8)
assert "emotion_valence" not in body # no PAD → neutral read (DEC-7)
assert body["voice"] == "Cora" # gateway default (DEC-8)
assert "emotion_valence" not in body # no p/a → neutral read (DEC-7)
def test_missing_text_returns_400(self) -> None:
from ratatoskr.web.server import create_app
app = create_app(_mock_client_factory(), tts_url=self._TTS)
resp = TestClient(app).post("/api/tts", json={"agent_id": "ratatoskr:donut"})
resp = TestClient(app).get("/api/tts", params={"agent_id": "ratatoskr:donut"})
assert resp.status_code == 400
assert resp.json()["error_code"] == "missing_text"
@@ -1530,56 +1516,32 @@ class TestTtsEndpoint:
respx.post(self._TTS).mock(return_value=httpx.Response(500, content=b"boom"))
app = create_app(_mock_client_factory(), tts_url=self._TTS)
resp = TestClient(app).post("/api/tts", json={"text": "hi"})
resp = TestClient(app).get("/api/tts", params={"text": "hi"})
assert resp.status_code == 503
assert resp.json()["error_code"] == "tts_unavailable"
@respx.mock
def test_non_wav_body_degrades_to_503(self) -> None:
from ratatoskr.web.server import create_app
def test_long_text_truncated_at_cap(self) -> None:
from ratatoskr.web.server import _TTS_MAX_TEXT_CHARS, create_app
respx.post(self._TTS).mock(
return_value=httpx.Response(200, content=b"<html>nope</html>")
route = respx.post(self._TTS).mock(
return_value=httpx.Response(200, content=self._WAV)
)
app = create_app(_mock_client_factory(), tts_url=self._TTS)
resp = TestClient(app).post("/api/tts", json={"text": "hi"})
assert resp.status_code == 503
resp = TestClient(app).get("/api/tts", params={"text": "word " * 1000})
assert resp.status_code == 200
body = json.loads(route.calls.last.request.content)
assert len(body["input"]) <= _TTS_MAX_TEXT_CHARS # truncated for the URL / lock
@respx.mock
def test_unhashable_agent_id_does_not_500(self) -> None:
# An unhashable agent_id ([] / {}) must not TypeError the voice-map lookup into
# a 500 — it coerces to None (default voice), matching the submit path's guard.
def test_malformed_pad_query_degrades_no_500(self) -> None:
from ratatoskr.web.server import create_app
route = respx.post(self._TTS).mock(
return_value=httpx.Response(200, content=self._WAV)
)
app = create_app(_mock_client_factory(), tts_url=self._TTS)
resp = TestClient(app).post("/api/tts", json={"text": "hi", "agent_id": []})
resp = TestClient(app).get("/api/tts", params={"text": "hi", "p": "notafloat", "a": "0.1"})
assert resp.status_code == 200
assert json.loads(route.calls.last.request.content)["voice"] == "Cora"
def test_text_too_large_returns_413(self) -> None:
from ratatoskr.web.server import _TTS_MAX_TEXT_CHARS, create_app
app = create_app(_mock_client_factory(), tts_url=self._TTS)
resp = TestClient(app).post(
"/api/tts", json={"text": "x" * (_TTS_MAX_TEXT_CHARS + 1)}
)
assert resp.status_code == 413
assert resp.json()["error_code"] == "text_too_large"
@respx.mock
def test_huge_int_pad_does_not_500(self) -> None:
# A malformed pad (huge-int → float() OverflowError in from_obj) must degrade to
# a neutral read, not 500 the endpoint.
from ratatoskr.web.server import create_app
respx.post(self._TTS).mock(return_value=httpx.Response(200, content=self._WAV))
app = create_app(_mock_client_factory(), tts_url=self._TTS)
resp = TestClient(app).post(
"/api/tts",
json={"text": "hi", "pad": {"pleasure": int("9" * 400), "arousal": 0}},
)
assert resp.status_code == 200
body = json.loads(route.calls.last.request.content)
assert "emotion_valence" not in body # malformed p → neutral read, not a 500