diff --git a/stacks/gateway-chat/conf/index.html b/stacks/gateway-chat/conf/index.html index de38ad2..2a1b032 100644 --- a/stacks/gateway-chat/conf/index.html +++ b/stacks/gateway-chat/conf/index.html @@ -160,7 +160,7 @@ async function streamSpeak(speech, tag){ const url = $('ttsUrl').value.trim().replace(/\/+$/,'').replace(/\/tts$/,'') + '/tts/stream'; const res = await fetch(url, { method:'POST', headers:{ 'Content-Type':'application/json' }, - body: JSON.stringify({ text: speech, voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 2400 }) + body: JSON.stringify({ text: speech, voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 3500 }) }); if (!res.ok || !res.body){ tag.textContent = ' 🔇' + res.status; return; } const reader = res.body.getReader(); diff --git a/stacks/mOrpheus/README.md b/stacks/mOrpheus/README.md index 153cf62..5dce7c5 100644 --- a/stacks/mOrpheus/README.md +++ b/stacks/mOrpheus/README.md @@ -24,9 +24,15 @@ lower time-to-first-audio is a future enhancement.) - `POST /tts` → `audio/wav`. Body: `{"text": "...", "voice": "baddy", "temperature": 0.6, "max_tokens": 1200, "repetition_penalty": 1.1}`. - - **Zero-shot clone:** add `"reference_audio_b64": ""` + `"reference_text": + - **Zero-shot clone (ad-hoc):** add `"reference_audio_b64": ""` + `"reference_text": ""`. Keep `repetition_penalty <= 1.1` for cloning (higher penalizes the in-context reference audio tokens and breaks generation). + - **Staged clone voices:** drop `.wav` + `.txt` (its transcript) into the voices + dir (`/home/lkraven/morpheus/voices/`); each is encoded to its reference block once at + startup, so `voice: ""` zero-shot clones it (e.g. `beatrice`). `GET /voices` lists them. + - `max_tokens` defaults to 3500 (~42 s), auto-clamped so prompt + gen never exceeds the + 4096 context (a cloning reference block is ~1,100 tokens). `repetition_penalty` 1.1 is + load-bearing — at 1.0 the model never emits end-of-speech and rambles to the cap. - `GET /voices`, `GET /health`, `GET /docs` (OpenAPI UI). **Expressive tags** (baddy is trained for these): ` diff --git a/stacks/mOrpheus/compose.yaml b/stacks/mOrpheus/compose.yaml index b8206f1..ebf2ed3 100644 --- a/stacks/mOrpheus/compose.yaml +++ b/stacks/mOrpheus/compose.yaml @@ -67,9 +67,12 @@ services: - SNAC_DEVICE=cpu - MORPHEUS_DEFAULT_VOICE=${MORPHEUS_DEFAULT_VOICE:-baddy} - MORPHEUS_VOICES=${MORPHEUS_VOICES:-baddy} + - MORPHEUS_VOICES_DIR=/voices # .wav + .txt => voice="" clones it + - MORPHEUS_MAX_LEN=${MORPHEUS_MAX_LEN:-4096} # must match engine --max-model-len (max_tokens clamp) volumes: - ${MORPHEUS_MODEL_DIR:-/home/lkraven/morpheus/models/mOrpheus}:/model:ro - ${SNAC_DIR:-/home/lkraven/morpheus/models/snac_24khz}:/snac:ro + - ${MORPHEUS_VOICES_DIR:-/home/lkraven/morpheus/voices}:/voices:ro ports: - "${MORPHEUS_TTS_PORT:-8299}:8000" healthcheck: diff --git a/stacks/mOrpheus/tts/app.py b/stacks/mOrpheus/tts/app.py index 7b8cfed..f443c42 100644 --- a/stacks/mOrpheus/tts/app.py +++ b/stacks/mOrpheus/tts/app.py @@ -30,6 +30,12 @@ DEFAULT_VOICE = os.environ.get("MORPHEUS_DEFAULT_VOICE", "baddy") VOICES = [v for v in os.environ.get("MORPHEUS_VOICES", "baddy").split(",") if v] AUDIO_MAX = 156937 AUDIO_BASE, SOS, EOS_SP, SOH, SOA, EOT, BOS = 128266, 128257, 128258, 128259, 128260, 128009, 128000 +MAX_CTX = int(os.environ.get("MORPHEUS_MAX_LEN", "4096")) # must match the engine --max-model-len + + +def _cap(prompt_ids: list[int], want: int) -> int: + # never let prompt + gen exceed the context (a cloning ref block is ~1100 tokens) + return max(64, min(want, MAX_CTX - len(prompt_ids) - 16)) tok = AutoTokenizer.from_pretrained(MODEL_DIR) snac_model = SNAC.from_pretrained(SNAC_DIR).to(SNAC_DEVICE).eval() @@ -45,7 +51,7 @@ class TTSReq(BaseModel): voice: str = DEFAULT_VOICE temperature: float = 0.6 top_p: float = 0.95 - max_tokens: int = 2400 # ~29s of audio; long lines were clipping at 1200 (~14.6s) + max_tokens: int = 3500 # ~42s of audio (auto-clamped to fit MAX_CTX; see _cap) repetition_penalty: float = 1.1 # LOAD-BEARING: rep 1.0 => model never stops (rambles to # the cap); rep 1.1 => clean end-of-speech. Keep <=1.1 for # cloning (higher penalizes the in-context ref audio tokens). @@ -72,10 +78,28 @@ def _encode_ref(wav_bytes: bytes, ref_text: str) -> list[int]: return [BOS, SOH] + tok(ref_text, add_special_tokens=False).input_ids + [EOT, SOA, SOS] + ids + [EOS_SP] +# Pre-staged clone voices: a /voices dir of .wav + .txt (its transcript). Each is +# encoded once to its Orpheus reference block at startup, so voice="" zero-shot clones it. +VOICES_DIR = os.environ.get("MORPHEUS_VOICES_DIR", "/voices") +CLONE_REFS = {} +if os.path.isdir(VOICES_DIR): + for fn in sorted(os.listdir(VOICES_DIR)): + if fn.endswith(".wav") and os.path.exists(os.path.join(VOICES_DIR, fn[:-4] + ".txt")): + name = fn[:-4] + try: + CLONE_REFS[name] = _encode_ref(open(os.path.join(VOICES_DIR, fn), "rb").read(), + open(os.path.join(VOICES_DIR, name + ".txt")).read().strip()) + print(f"[voices] staged clone voice '{name}'", flush=True) + except Exception as e: + print(f"[voices] FAILED to stage '{name}': {e}", flush=True) + + def _build_prompt(req: TTSReq) -> list[int]: if req.reference_audio_b64 and req.reference_text: ref = _encode_ref(base64.b64decode(req.reference_audio_b64), req.reference_text) return ref + [SOH] + tok(req.text, add_special_tokens=False).input_ids + [EOT, SOA] + if req.voice in CLONE_REFS: # pre-staged clone voice (e.g. beatrice) + return CLONE_REFS[req.voice] + [SOH] + tok(req.text, add_special_tokens=False).input_ids + [EOT, SOA] return [SOH] + tok(f"{req.voice}: {req.text}").input_ids + [EOT, SOA] @@ -101,18 +125,18 @@ def _decode(ids: list[int]): @app.get("/health") def health(): - return {"status": "ok", "voices": VOICES, "default": DEFAULT_VOICE, "engine": VLLM_URL} + return {"status": "ok", "voices": VOICES + list(CLONE_REFS), "default": DEFAULT_VOICE, "engine": VLLM_URL} @app.get("/voices") def voices(): - return {"voices": VOICES, "default": DEFAULT_VOICE} + return {"voices": VOICES + list(CLONE_REFS), "cloned": list(CLONE_REFS), "default": DEFAULT_VOICE} @app.post("/tts") def tts(req: TTSReq): prompt_ids = _build_prompt(req) - payload = {"model": VLLM_MODEL, "prompt": prompt_ids, "max_tokens": req.max_tokens, + payload = {"model": VLLM_MODEL, "prompt": prompt_ids, "max_tokens": _cap(prompt_ids, req.max_tokens), "temperature": req.temperature, "top_p": req.top_p, "skip_special_tokens": False, "stop_token_ids": [EOS_SP], "repetition_penalty": req.repetition_penalty} try: @@ -168,7 +192,7 @@ def _pcm16(arr) -> bytes: @app.post("/tts/stream") def tts_stream(req: TTSReq): prompt_ids = _build_prompt(req) - payload = {"model": VLLM_MODEL, "prompt": prompt_ids, "max_tokens": req.max_tokens, + payload = {"model": VLLM_MODEL, "prompt": prompt_ids, "max_tokens": _cap(prompt_ids, req.max_tokens), "temperature": req.temperature, "top_p": req.top_p, "skip_special_tokens": False, "stop_token_ids": [EOS_SP], "repetition_penalty": req.repetition_penalty, "stream": True} CTX, CHUNK = 2, 6 # context frames each side, frames emitted per decode diff --git a/tools/gateway-chat.html b/tools/gateway-chat.html index de38ad2..2a1b032 100644 --- a/tools/gateway-chat.html +++ b/tools/gateway-chat.html @@ -160,7 +160,7 @@ async function streamSpeak(speech, tag){ const url = $('ttsUrl').value.trim().replace(/\/+$/,'').replace(/\/tts$/,'') + '/tts/stream'; const res = await fetch(url, { method:'POST', headers:{ 'Content-Type':'application/json' }, - body: JSON.stringify({ text: speech, voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 2400 }) + body: JSON.stringify({ text: speech, voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 3500 }) }); if (!res.ok || !res.body){ tag.textContent = ' 🔇' + res.status; return; } const reader = res.body.getReader();