From cd92b851577c66e2def3770a4ee163aa15293994 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Fri, 19 Jun 2026 22:58:55 -0700 Subject: [PATCH] feat(omnivoice): tune streaming defaults (16-step + aggressive packing) Empirical follow-up to the streaming /tts smoke test on the 3090. OmniVoice is diffusion: a ~fixed per-call overhead (~1.5s at 32 steps, ~0.7s at 16) dominates regardless of chunk length, so the upstream-claimed 40x RTF does NOT hold here (measured ~2.8x/32-step, ~5.6x/16-step) and the chatterbox- tuned scheduler over-chunks and starves. - Streaming /tts defaults to num_step=16 (TTFA ~1.5s -> ~0.7s); batch /v1/audio/speech stays num_step=32 for quality. Per-request override intact. - Scheduler prior raised to rtf_prior=20 (env OMNIVOICE_STREAM_RTF_PRIOR, wired through compose + .env.example) so it packs whole-text-minus-first- sentence into a few chunks: validated ~3 chunks, no starvation, total wall ~= one-shot, less per-chunk silence padding. - Docs corrected: the "sub-second / 40x" claims were wrong; streaming has a diffusion TTFA floor (~0.7s) and wins mainly on long replies. chatterbox- fast (autoregressive, ~0.5s TTFA) stays the lowest-latency front-end; OmniVoice is the multilingual / voice-design complement. --- stacks/omnivoice/.env.example | 5 +++++ stacks/omnivoice/README.md | 29 +++++++++++++++++++++++------ stacks/omnivoice/app.py | 23 +++++++++++++++++++---- stacks/omnivoice/compose.yaml | 3 +++ 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/stacks/omnivoice/.env.example b/stacks/omnivoice/.env.example index b542d7d..43f37b7 100644 --- a/stacks/omnivoice/.env.example +++ b/stacks/omnivoice/.env.example @@ -18,3 +18,8 @@ OMNIVOICE_VERSION= # Persistent HF weight cache + reference-voice staging on /worktank. OMNIVOICE_CACHE_DIR=/worktank/omnivoice/hf_cache OMNIVOICE_VOICES_DIR=/worktank/omnivoice/voices + +# Streaming /tts scheduler prior. High = pack aggressively (OmniVoice is diffusion +# with a ~fixed per-call overhead; low priors over-chunk and starve). 20 is +# validated clean on the 3090. Per-request `rtf_prior` overrides this. +OMNIVOICE_STREAM_RTF_PRIOR=20 diff --git a/stacks/omnivoice/README.md b/stacks/omnivoice/README.md index 48bc69b..f2f7046 100644 --- a/stacks/omnivoice/README.md +++ b/stacks/omnivoice/README.md @@ -36,15 +36,32 @@ reference), so per-request latency is just generation. The full generation surface is exposed: zero-shot **clone** (`voice`) and/or voice-**design** (`instruct`), plus `language` / `speed` / `duration` and the diffusion knobs. -### Streaming — sub-second time-to-first-audio +### Streaming — earlier first-audio (with a diffusion floor) `POST /tts` (`stream=true`, default) runs the **adaptive buffer-ratchet scheduler** vendored from chatterbox-fast ([`scheduler.py`](scheduler.py)): it -emits the first sentence immediately and ratchets chunk size up on OmniVoice's -~40× realtime headroom, so a live consumer hears speech start in ~tens of ms -instead of waiting for the whole utterance. `stream=false` is a whole-text -one-shot for A/B. Scheduler tunables (`margin`, `margin_first`, `rtf_prior`, -`sec_per_char_prior`) are per-request overrides. +emits the first sentence immediately, then packs the rest into a few chunks so a +live consumer hears speech start sooner than waiting for the whole utterance. +`stream=false` is a whole-text one-shot for A/B. + +**Measured reality (3090, not the upstream-claimed 40× RTF):** OmniVoice is a +diffusion model, so each `generate()` call has a **~fixed per-call overhead** +(~1.5 s at `num_step=32`, ~0.7 s at 16) that sets a **time-to-first-audio +floor** — short and long chunks cost nearly the same. Server-side TTFA is +therefore ~0.7 s (streaming default, 16 steps), **not** sub-second-at-full- +quality. Effective RTF is ~2.8× (32 steps) / ~5.6× (16 steps). The win over +one-shot is small for short replies and grows with length (one-shot TTFA scales +with the whole utterance; streaming stays ~flat at the first-sentence cost). +For absolute-lowest TTFA, **chatterbox-fast** (autoregressive, ~0.5 s) remains +the better front-end; OmniVoice is the multilingual / voice-design complement. + +Defaults tuned for this: **streaming `num_step=16`** (batch `/v1/audio/speech` +stays 32 for quality), and an **aggressive packing prior** (`rtf_prior=20`, env +`OMNIVOICE_STREAM_RTF_PRIOR`) — diffusion's fixed overhead makes the chatterbox +default over-chunk and starve, so we pack whole-text-minus-first-sentence into a +few chunks (validated: ~3 chunks, no starvation, total ≈ one-shot). Scheduler +tunables (`margin`, `margin_first`, `rtf_prior`, `sec_per_char_prior`) and +`num_step` are per-request overrides. `scheduler.py` is a **vendored byte-faithful copy** (not a dependency) of chatterbox-fast's pure-Python, torch-free scheduler — see its header for the diff --git a/stacks/omnivoice/app.py b/stacks/omnivoice/app.py index a1e7bc4..685bf51 100644 --- a/stacks/omnivoice/app.py +++ b/stacks/omnivoice/app.py @@ -10,9 +10,11 @@ Two consumption modes: - BATCH (asset-engine / OpenAI-compat): POST /v1/audio/speech -> one WAV blob. - STREAM (live speech-to-speech chat engines): POST /tts -> chunked PCM, driven by the vendored adaptive buffer-ratchet scheduler (scheduler.py, from - chatterbox-fast). Emits the first sentence immediately for sub-second - time-to-first-audio, then ratchets chunk size up on OmniVoice's ~40x realtime - headroom. Wire-compatible with chatterbox-fast's /tts (both 24 kHz mono s16le). + chatterbox-fast). Emits the first sentence immediately so first-audio comes + sooner than one-shot, then packs the rest into a few chunks. NB: OmniVoice is + diffusion, so a ~fixed per-call overhead sets a TTFA floor (~0.7s at 16 steps + on the 3090, NOT sub-second); the win grows with utterance length. Wire- + compatible with chatterbox-fast's /tts (both 24 kHz mono s16le). All text is run through the language-safe sanitizer (sanitize.py) before synthesis on BOTH endpoints — strips markdown / LLM artifacts / control tokens without the @@ -76,6 +78,14 @@ CKPT = os.environ.get("OMNIVOICE_CKPT", "k2-fsa/OmniVoice") VOICES_DIR = os.environ.get("OMNIVOICE_VOICES_DIR", "/app/voices") ASR_MODEL = os.environ.get("OMNIVOICE_ASR_MODEL", "openai/whisper-large-v3-turbo") +# Streaming scheduler prior. OmniVoice is diffusion: a ~fixed per-call overhead +# dominates (short and long chunks cost ~the same), so the chatterbox default +# (rtf_prior=3.4) over-chunks and STARVES — each extra chunk re-pays the fixed +# cost and adds boundary silence. A high prior packs whole-text-minus-first- +# sentence into a few chunks (validated on the 3090: ~3 chunks, no starvation, +# total ≈ one-shot). Per-request `rtf_prior` still overrides this. +OMNIVOICE_STREAM_RTF_PRIOR = float(os.environ.get("OMNIVOICE_STREAM_RTF_PRIOR", "20")) + app = FastAPI(title="OmniVoice TTS (asset-engine + streaming wrapper)") MODEL: Optional[OmniVoice] = None @@ -99,7 +109,7 @@ class GenParams(BaseModel): language: Optional[str] = "Auto" # "Auto" -> auto-detect speed: Optional[float] = None # 0.5–1.5; ignored if duration set duration: Optional[float] = None # fixed seconds; overrides speed - num_step: int = 32 # 4–64 diffusion steps + num_step: int = 32 # 4–64 diffusion steps (batch=32; /tts overrides to 16) guidance_scale: float = 2.0 # 0.0–4.0 CFG denoise: bool = True preprocess_prompt: bool = True @@ -120,6 +130,10 @@ class SpeechRequest(GenParams): class TTSStreamRequest(GenParams): """Streaming /tts request — chatterbox-fast-compatible wire protocol.""" + # Streaming defaults to FEWER diffusion steps than batch (32): halves the + # ~per-call diffusion overhead (server-side TTFA ~1.5s -> ~0.7s on the 3090) + # at some quality cost. Override per-request for the quality/latency trade. + num_step: int = 16 format: Literal["pcm", "wav"] = "pcm" # raw s16le PCM (default) or open-ended WAV stream: bool = True # False -> whole-text one-shot (A/B vs stream) # Scheduler overrides (None -> ChunkConfig defaults; see scheduler.py). @@ -224,6 +238,7 @@ def _wav_header(sr: int, data_len: Optional[int] = None) -> bytes: def _chunk_config(req: TTSStreamRequest) -> ChunkConfig: cfg = ChunkConfig() + cfg.rtf_prior = OMNIVOICE_STREAM_RTF_PRIOR # diffusion-aware default (pack aggressively) if req.margin is not None: cfg.margin = req.margin if req.margin_first is not None: diff --git a/stacks/omnivoice/compose.yaml b/stacks/omnivoice/compose.yaml index 1a4fe84..487dfce 100644 --- a/stacks/omnivoice/compose.yaml +++ b/stacks/omnivoice/compose.yaml @@ -35,6 +35,9 @@ services: environment: - NVIDIA_VISIBLE_DEVICES=${OMNIVOICE_GPU_DEVICES:-0} - HF_HOME=/app/hf_cache + # Streaming /tts scheduler prior — high = pack aggressively (diffusion has a + # ~fixed per-call overhead; low priors over-chunk and starve). See app.py. + - OMNIVOICE_STREAM_RTF_PRIOR=${OMNIVOICE_STREAM_RTF_PRIOR:-20} volumes: - ${OMNIVOICE_CACHE_DIR}:/app/hf_cache - ${OMNIVOICE_VOICES_DIR}:/app/voices