From a1f3023f70a3e98e5aeda7bd741751638f07b9dd Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Thu, 9 Jul 2026 01:49:45 -0700 Subject: [PATCH] feat(gateway-chat): pre-chunk quoted text by sentence, play serially MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split the quoted dialogue into sentences and stream each as its own short /tts/stream request (max_tokens 900), queued back-to-back on one shared AudioContext clock (speechHead) so playback is gapless and in order. First sentence starts fast; each chunk is short so it generates cleanly (no ramble/cap risk); the next sentence generates while the current plays. A newer reply supersedes via the ttsGen counter; 🔊 replays. --- stacks/gateway-chat/conf/index.html | 41 +++++++++++++++++++---------- tools/gateway-chat.html | 41 +++++++++++++++++++---------- 2 files changed, 54 insertions(+), 28 deletions(-) diff --git a/stacks/gateway-chat/conf/index.html b/stacks/gateway-chat/conf/index.html index c29dbd3..ae593a9 100644 --- a/stacks/gateway-chat/conf/index.html +++ b/stacks/gateway-chat/conf/index.html @@ -147,6 +147,7 @@ function showErr(text){ const m = addMsg('error','err'); m.body.className = 'err // --- mOrpheus TTS: streaming-decode auto-voice of quoted text (Web Audio, low TTFA) --- let audioCtx = null; // shared AudioContext let ttsGen = 0; // generation counter — a newer reply's stream supersedes older ones +let speechHead = 0; // shared playback clock — sentences queue onto it back-to-back function primeAudio(){ // browsers suspend the AudioContext until a user gesture; resume on ANY if (!audioCtx) audioCtx = new (window.AudioContext || window.webkitAudioContext)(); if (audioCtx.state === 'suspended') audioCtx.resume(); // interaction so async playback isn't silent @@ -159,18 +160,20 @@ function extractQuotes(text){ while ((m = re.exec(text)) !== null){ const q = (m[1] || m[2] || '').trim(); if (q) out.push(q); } return out; } -async function streamSpeak(speech, tag){ - const mine = ++ttsGen; // cancels any in-flight stream - if (!audioCtx) audioCtx = new (window.AudioContext || window.webkitAudioContext)(); - if (audioCtx.state === 'suspended') await audioCtx.resume(); +function splitSentences(t){ + return (t.match(/[^.!?…]+[.!?…]+["”’')\]]*\s*|[^.!?…]+$/g) || [t]).map(s => s.trim()).filter(Boolean); +} +// stream ONE sentence and queue its PCM onto the shared speechHead clock; resolves when the +// sentence's audio is fully received (it plays on while the NEXT sentence starts generating). +async function streamOne(sentence, mine){ const url = $('ttsUrl').value.trim().replace(/\/+$/,'').replace(/\/tts$/,'') + '/tts/stream'; const res = await fetch(url, { method:'POST', headers:{ 'Content-Type':'application/json' }, - body: JSON.stringify({ text: speech, voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 3500 }) + body: JSON.stringify({ text: sentence, voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 900 }) }); - if (!res.ok || !res.body){ tag.textContent = ' 🔇' + res.status; return; } + if (!res.ok || !res.body) return false; const reader = res.body.getReader(); - let playhead = audioCtx.currentTime + 0.12, leftover = new Uint8Array(0); + let leftover = new Uint8Array(0); for(;;){ const { done, value } = await reader.read(); if (done || mine !== ttsGen) break; // stop if a newer reply took over @@ -183,21 +186,31 @@ async function streamSpeak(speech, tag){ for (let i = 0; i < N; i++) f32[i] = dv.getInt16(i * 2, true) / 32768; const abuf = audioCtx.createBuffer(1, N, 24000); abuf.getChannelData(0).set(f32); const src = audioCtx.createBufferSource(); src.buffer = abuf; src.connect(audioCtx.destination); - if (playhead < audioCtx.currentTime) playhead = audioCtx.currentTime + 0.02; // underrun catch-up - src.start(playhead); playhead += abuf.duration; + if (speechHead < audioCtx.currentTime + 0.05) speechHead = audioCtx.currentTime + 0.05; // underrun catch-up + src.start(speechHead); speechHead += abuf.duration; // queue back-to-back on the shared clock } - if (mine === ttsGen) tag.textContent = ' 🔈'; + return true; } async function speakQuotes(text, msg){ if (!$('ttsOn').checked) return; const quotes = extractQuotes(text); if (!quotes.length) return; - const speech = quotes.join(' '); + const sentences = splitSentences(quotes.join(' ')); // pre-chunk quoted text by sentence const tag = document.createElement('span'); tag.textContent = ' 🔊'; - tag.style.cursor = 'pointer'; tag.title = 'mOrpheus — click to replay'; - tag.onclick = () => streamSpeak(speech, tag).catch(() => tag.textContent = ' 🔇'); + tag.style.cursor = 'pointer'; tag.title = 'mOrpheus — ' + sentences.length + ' sentence(s), click to replay'; + const play = async () => { + const mine = ++ttsGen; // supersede any in-flight playback + primeAudio(); if (audioCtx.state === 'suspended') await audioCtx.resume(); + speechHead = audioCtx.currentTime + 0.12; // reset the shared clock for this reply + for (const s of sentences){ // play sentences serially, in order + if (mine !== ttsGen) return; + if (!(await streamOne(s, mine))){ tag.textContent = ' 🔇'; return; } + } + if (mine === ttsGen) tag.textContent = ' 🔈'; + }; + tag.onclick = () => play().catch(() => tag.textContent = ' 🔇'); msg.wrap.querySelector('.who').append(tag); - try { await streamSpeak(speech, tag); } catch (e){ tag.textContent = ' 🔇'; } + try { await play(); } catch (e){ tag.textContent = ' 🔇'; } } async function send(){ diff --git a/tools/gateway-chat.html b/tools/gateway-chat.html index c29dbd3..ae593a9 100644 --- a/tools/gateway-chat.html +++ b/tools/gateway-chat.html @@ -147,6 +147,7 @@ function showErr(text){ const m = addMsg('error','err'); m.body.className = 'err // --- mOrpheus TTS: streaming-decode auto-voice of quoted text (Web Audio, low TTFA) --- let audioCtx = null; // shared AudioContext let ttsGen = 0; // generation counter — a newer reply's stream supersedes older ones +let speechHead = 0; // shared playback clock — sentences queue onto it back-to-back function primeAudio(){ // browsers suspend the AudioContext until a user gesture; resume on ANY if (!audioCtx) audioCtx = new (window.AudioContext || window.webkitAudioContext)(); if (audioCtx.state === 'suspended') audioCtx.resume(); // interaction so async playback isn't silent @@ -159,18 +160,20 @@ function extractQuotes(text){ while ((m = re.exec(text)) !== null){ const q = (m[1] || m[2] || '').trim(); if (q) out.push(q); } return out; } -async function streamSpeak(speech, tag){ - const mine = ++ttsGen; // cancels any in-flight stream - if (!audioCtx) audioCtx = new (window.AudioContext || window.webkitAudioContext)(); - if (audioCtx.state === 'suspended') await audioCtx.resume(); +function splitSentences(t){ + return (t.match(/[^.!?…]+[.!?…]+["”’')\]]*\s*|[^.!?…]+$/g) || [t]).map(s => s.trim()).filter(Boolean); +} +// stream ONE sentence and queue its PCM onto the shared speechHead clock; resolves when the +// sentence's audio is fully received (it plays on while the NEXT sentence starts generating). +async function streamOne(sentence, mine){ const url = $('ttsUrl').value.trim().replace(/\/+$/,'').replace(/\/tts$/,'') + '/tts/stream'; const res = await fetch(url, { method:'POST', headers:{ 'Content-Type':'application/json' }, - body: JSON.stringify({ text: speech, voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 3500 }) + body: JSON.stringify({ text: sentence, voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 900 }) }); - if (!res.ok || !res.body){ tag.textContent = ' 🔇' + res.status; return; } + if (!res.ok || !res.body) return false; const reader = res.body.getReader(); - let playhead = audioCtx.currentTime + 0.12, leftover = new Uint8Array(0); + let leftover = new Uint8Array(0); for(;;){ const { done, value } = await reader.read(); if (done || mine !== ttsGen) break; // stop if a newer reply took over @@ -183,21 +186,31 @@ async function streamSpeak(speech, tag){ for (let i = 0; i < N; i++) f32[i] = dv.getInt16(i * 2, true) / 32768; const abuf = audioCtx.createBuffer(1, N, 24000); abuf.getChannelData(0).set(f32); const src = audioCtx.createBufferSource(); src.buffer = abuf; src.connect(audioCtx.destination); - if (playhead < audioCtx.currentTime) playhead = audioCtx.currentTime + 0.02; // underrun catch-up - src.start(playhead); playhead += abuf.duration; + if (speechHead < audioCtx.currentTime + 0.05) speechHead = audioCtx.currentTime + 0.05; // underrun catch-up + src.start(speechHead); speechHead += abuf.duration; // queue back-to-back on the shared clock } - if (mine === ttsGen) tag.textContent = ' 🔈'; + return true; } async function speakQuotes(text, msg){ if (!$('ttsOn').checked) return; const quotes = extractQuotes(text); if (!quotes.length) return; - const speech = quotes.join(' '); + const sentences = splitSentences(quotes.join(' ')); // pre-chunk quoted text by sentence const tag = document.createElement('span'); tag.textContent = ' 🔊'; - tag.style.cursor = 'pointer'; tag.title = 'mOrpheus — click to replay'; - tag.onclick = () => streamSpeak(speech, tag).catch(() => tag.textContent = ' 🔇'); + tag.style.cursor = 'pointer'; tag.title = 'mOrpheus — ' + sentences.length + ' sentence(s), click to replay'; + const play = async () => { + const mine = ++ttsGen; // supersede any in-flight playback + primeAudio(); if (audioCtx.state === 'suspended') await audioCtx.resume(); + speechHead = audioCtx.currentTime + 0.12; // reset the shared clock for this reply + for (const s of sentences){ // play sentences serially, in order + if (mine !== ttsGen) return; + if (!(await streamOne(s, mine))){ tag.textContent = ' 🔇'; return; } + } + if (mine === ttsGen) tag.textContent = ' 🔈'; + }; + tag.onclick = () => play().catch(() => tag.textContent = ' 🔇'); msg.wrap.querySelector('.who').append(tag); - try { await streamSpeak(speech, tag); } catch (e){ tag.textContent = ' 🔇'; } + try { await play(); } catch (e){ tag.textContent = ' 🔇'; } } async function send(){