vh/esh-pfi-infrastructure
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(morpheus,gateway-chat): streaming decode — TTFA ~4.5s -> ~0.8s

Browse Source 
Wrapper gains POST /tts/stream: reads the vLLM token stream, decodes SNAC in WINDOWED
CHUNKS (every 6 frames, decode [2 ctx | 6 | 2 ctx] and emit only the middle 6 — context
both sides => seamless), and streams raw PCM16 (24kHz mono) as it generates. Windowed
(not per-frame) because per-frame CPU decode's per-call overhead x ~60 frames serialized
to ~7s (RTF 2.2); windowed keeps up (RTF ~0.97). Whole-clip /tts kept for non-browser use.

gateway-chat plays the stream via the Web Audio API (fetch reader -> int16->float32 ->
scheduled AudioBufferSourceNodes on a running clock; a new reply supersedes the prior
stream via a generation counter; 🔊 replays). Measured: TTFA 0.80s (was ~4.5s whole-clip),
RTF 0.97, full-duration match. CORS already covers the new route.

Deployed: tts rebuilt on irv-ml1, page pushed to ana-docker.
This commit is contained in:
vh
2026-07-09 01:14:25 -07:00
parent c948013a36
commit da7682969b
3 changed files with 165 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
stacks/gateway-chat/conf/index.html
+37 -16
View File
@@ -144,33 +144,54 @@ function addMsg(who, cls){
}
function showErr(text){ const m = addMsg('error','err'); m.body.className = 'err'; m.body.textContent = text; }
// --- mOrpheus TTS: auto-voice quoted text from a completed assistant reply ---
let ttsAudio = null; // current clip, so a new reply interrupts the previous one
// --- mOrpheus TTS: streaming-decode auto-voice of quoted text (Web Audio, low TTFA) ---
let audioCtx = null; // shared AudioContext
let ttsGen = 0; // generation counter — a newer reply's stream supersedes older ones
function extractQuotes(text){
const re = /“([^”]+)”|"([^"]+)"/g; // typographic "…" or straight "…"
const out = []; let m;
while ((m = re.exec(text)) !== null){ const q = (m[1] || m[2] || '').trim(); if (q) out.push(q); }
return out;
}
async function streamSpeak(speech, tag){
const mine = ++ttsGen; // cancels any in-flight stream
if (!audioCtx) audioCtx = new (window.AudioContext || window.webkitAudioContext)();
if (audioCtx.state === 'suspended') await audioCtx.resume();
const url = $('ttsUrl').value.trim().replace(/\/+$/,'').replace(/\/tts$/,'') + '/tts/stream';
const res = await fetch(url, {
method:'POST', headers:{ 'Content-Type':'application/json' },
body: JSON.stringify({ text: speech, voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 1200 })
});
if (!res.ok || !res.body){ tag.textContent = ' 🔇' + res.status; return; }
const reader = res.body.getReader();
let playhead = audioCtx.currentTime + 0.12, leftover = new Uint8Array(0);
for(;;){
const { done, value } = await reader.read();
if (done || mine !== ttsGen) break; // stop if a newer reply took over
let bytes = new Uint8Array(leftover.length + value.length);
bytes.set(leftover); bytes.set(value, leftover.length);
const n = bytes.length - (bytes.length % 2); // whole int16 samples only
leftover = bytes.slice(n);
if (!n) continue;
const dv = new DataView(bytes.buffer, 0, n), N = n / 2, f32 = new Float32Array(N);
for (let i = 0; i < N; i++) f32[i] = dv.getInt16(i * 2, true) / 32768;
const abuf = audioCtx.createBuffer(1, N, 24000); abuf.getChannelData(0).set(f32);
const src = audioCtx.createBufferSource(); src.buffer = abuf; src.connect(audioCtx.destination);
if (playhead < audioCtx.currentTime) playhead = audioCtx.currentTime + 0.02; // underrun catch-up
src.start(playhead); playhead += abuf.duration;
}
if (mine === ttsGen) tag.textContent = ' 🔈';
}
async function speakQuotes(text, msg){
if (!$('ttsOn').checked) return;
const quotes = extractQuotes(text);
if (!quotes.length) return;
const speech = quotes.join(' ');
const tag = document.createElement('span'); tag.textContent = ' 🔊';
tag.style.cursor = 'pointer'; tag.title = 'mOrpheus'; msg.wrap.querySelector('.who').append(tag);
try {
const res = await fetch($('ttsUrl').value.trim(), {
method:'POST', headers:{ 'Content-Type':'application/json' },
body: JSON.stringify({ text: quotes.join(' '), voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 1200 })
});
if (!res.ok){ tag.textContent = ' 🔇' + res.status; return; }
const url = URL.createObjectURL(await res.blob());
if (ttsAudio) ttsAudio.pause();
ttsAudio = new Audio(url);
tag.onclick = () => ttsAudio.play(); // click 🔊 to replay
ttsAudio.onended = () => tag.textContent = ' 🔈';
ttsAudio.play();
} catch (e){ tag.textContent = ' 🔇'; }
tag.style.cursor = 'pointer'; tag.title = 'mOrpheus — click to replay';
tag.onclick = () => streamSpeak(speech, tag).catch(() => tag.textContent = ' 🔇');
msg.wrap.querySelector('.who').append(tag);
try { await streamSpeak(speech, tag); } catch (e){ tag.textContent = ' 🔇'; }
}
async function send(){