feat(gateway-chat): pre-chunk quoted text by sentence, play serially
Split the quoted dialogue into sentences and stream each as its own short /tts/stream
request (max_tokens 900), queued back-to-back on one shared AudioContext clock (speechHead)
so playback is gapless and in order. First sentence starts fast; each chunk is short so it
generates cleanly (no ramble/cap risk); the next sentence generates while the current plays.
A newer reply supersedes via the ttsGen counter; 🔊 replays.
This commit is contained in:
+27
-14
@@ -147,6 +147,7 @@ function showErr(text){ const m = addMsg('error','err'); m.body.className = 'err
|
||||
// --- mOrpheus TTS: streaming-decode auto-voice of quoted text (Web Audio, low TTFA) ---
|
||||
let audioCtx = null; // shared AudioContext
|
||||
let ttsGen = 0; // generation counter — a newer reply's stream supersedes older ones
|
||||
let speechHead = 0; // shared playback clock — sentences queue onto it back-to-back
|
||||
function primeAudio(){ // browsers suspend the AudioContext until a user gesture; resume on ANY
|
||||
if (!audioCtx) audioCtx = new (window.AudioContext || window.webkitAudioContext)();
|
||||
if (audioCtx.state === 'suspended') audioCtx.resume(); // interaction so async playback isn't silent
|
||||
@@ -159,18 +160,20 @@ function extractQuotes(text){
|
||||
while ((m = re.exec(text)) !== null){ const q = (m[1] || m[2] || '').trim(); if (q) out.push(q); }
|
||||
return out;
|
||||
}
|
||||
async function streamSpeak(speech, tag){
|
||||
const mine = ++ttsGen; // cancels any in-flight stream
|
||||
if (!audioCtx) audioCtx = new (window.AudioContext || window.webkitAudioContext)();
|
||||
if (audioCtx.state === 'suspended') await audioCtx.resume();
|
||||
function splitSentences(t){
|
||||
return (t.match(/[^.!?…]+[.!?…]+["”’')\]]*\s*|[^.!?…]+$/g) || [t]).map(s => s.trim()).filter(Boolean);
|
||||
}
|
||||
// stream ONE sentence and queue its PCM onto the shared speechHead clock; resolves when the
|
||||
// sentence's audio is fully received (it plays on while the NEXT sentence starts generating).
|
||||
async function streamOne(sentence, mine){
|
||||
const url = $('ttsUrl').value.trim().replace(/\/+$/,'').replace(/\/tts$/,'') + '/tts/stream';
|
||||
const res = await fetch(url, {
|
||||
method:'POST', headers:{ 'Content-Type':'application/json' },
|
||||
body: JSON.stringify({ text: speech, voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 3500 })
|
||||
body: JSON.stringify({ text: sentence, voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 900 })
|
||||
});
|
||||
if (!res.ok || !res.body){ tag.textContent = ' 🔇' + res.status; return; }
|
||||
if (!res.ok || !res.body) return false;
|
||||
const reader = res.body.getReader();
|
||||
let playhead = audioCtx.currentTime + 0.12, leftover = new Uint8Array(0);
|
||||
let leftover = new Uint8Array(0);
|
||||
for(;;){
|
||||
const { done, value } = await reader.read();
|
||||
if (done || mine !== ttsGen) break; // stop if a newer reply took over
|
||||
@@ -183,21 +186,31 @@ async function streamSpeak(speech, tag){
|
||||
for (let i = 0; i < N; i++) f32[i] = dv.getInt16(i * 2, true) / 32768;
|
||||
const abuf = audioCtx.createBuffer(1, N, 24000); abuf.getChannelData(0).set(f32);
|
||||
const src = audioCtx.createBufferSource(); src.buffer = abuf; src.connect(audioCtx.destination);
|
||||
if (playhead < audioCtx.currentTime) playhead = audioCtx.currentTime + 0.02; // underrun catch-up
|
||||
src.start(playhead); playhead += abuf.duration;
|
||||
if (speechHead < audioCtx.currentTime + 0.05) speechHead = audioCtx.currentTime + 0.05; // underrun catch-up
|
||||
src.start(speechHead); speechHead += abuf.duration; // queue back-to-back on the shared clock
|
||||
}
|
||||
if (mine === ttsGen) tag.textContent = ' 🔈';
|
||||
return true;
|
||||
}
|
||||
async function speakQuotes(text, msg){
|
||||
if (!$('ttsOn').checked) return;
|
||||
const quotes = extractQuotes(text);
|
||||
if (!quotes.length) return;
|
||||
const speech = quotes.join(' ');
|
||||
const sentences = splitSentences(quotes.join(' ')); // pre-chunk quoted text by sentence
|
||||
const tag = document.createElement('span'); tag.textContent = ' 🔊';
|
||||
tag.style.cursor = 'pointer'; tag.title = 'mOrpheus — click to replay';
|
||||
tag.onclick = () => streamSpeak(speech, tag).catch(() => tag.textContent = ' 🔇');
|
||||
tag.style.cursor = 'pointer'; tag.title = 'mOrpheus — ' + sentences.length + ' sentence(s), click to replay';
|
||||
const play = async () => {
|
||||
const mine = ++ttsGen; // supersede any in-flight playback
|
||||
primeAudio(); if (audioCtx.state === 'suspended') await audioCtx.resume();
|
||||
speechHead = audioCtx.currentTime + 0.12; // reset the shared clock for this reply
|
||||
for (const s of sentences){ // play sentences serially, in order
|
||||
if (mine !== ttsGen) return;
|
||||
if (!(await streamOne(s, mine))){ tag.textContent = ' 🔇'; return; }
|
||||
}
|
||||
if (mine === ttsGen) tag.textContent = ' 🔈';
|
||||
};
|
||||
tag.onclick = () => play().catch(() => tag.textContent = ' 🔇');
|
||||
msg.wrap.querySelector('.who').append(tag);
|
||||
try { await streamSpeak(speech, tag); } catch (e){ tag.textContent = ' 🔇'; }
|
||||
try { await play(); } catch (e){ tag.textContent = ' 🔇'; }
|
||||
}
|
||||
|
||||
async function send(){
|
||||
|
||||
Reference in New Issue
Block a user