vh/esh-pfi-infrastructure
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(gateway-chat): auto-voice quoted dialogue via mOrpheus TTS

Browse Source 
Gateway-chat now auto-plays quoted text from each assistant reply through the mOrpheus
TTS endpoint. Sidebar gains a 🔊 toggle + endpoint/voice fields (persist in localStorage,
prefilled to irv-ml1:8299 / baddy). On reply-complete, straight and typographic double
quotes are extracted, joined, POSTed to /tts, and the returned WAV plays (click 🔊 to
replay; a new reply interrupts the prior clip).

Requires CORS on the wrapper (page served from ana-docker:8091 fetches irv-ml1:8299
cross-origin) — added CORSMiddleware(allow_origins=[*]) to the mOrpheus tts app (internal-
only endpoint). Verified end-to-end: preflight + POST return ACAO=*, valid 24kHz WAV.

Deployed: tts container rebuilt/recreated on irv-ml1; page pushed to ana-docker conf
(bind-mounted, live on next request).
This commit is contained in:
Vuong Hoang
2026-07-09 00:58:13 -07:00
parent 01eedd8d27
commit c948013a36
3 changed files with 87 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
stacks/gateway-chat/conf/index.html
+41 -2
View File
@@ -67,7 +67,11 @@
<div><label>Temperature</label><input id="temp" type="number" step="0.05" value="1"></div>
<div><label>Max tokens</label><input id="max" type="number" step="1" value="1024"></div>
</div>
<div class="hint">Settings persist locally. No <code>tools</code> field is ever sent. Reasoning models show their <code>reasoning_content</code> above the answer.</div>
<label style="margin-top:.5rem">🔊 mOrpheus voice (quoted text)</label>
<label class="hint" style="display:flex;gap:.45rem;align-items:center;margin:.15rem 0"><input id="ttsOn" type="checkbox" style="width:auto"> Auto-play quoted dialogue</label>
<input id="ttsUrl" placeholder="http://10.100.79.3:8299/tts">
<input id="ttsVoice" placeholder="baddy">
<div class="hint">Settings persist locally. No <code>tools</code> field is ever sent. Reasoning models show their <code>reasoning_content</code> above the answer. With 🔊 on, quoted text in each reply auto-plays via mOrpheus (irv-ml1, internal).</div>
</aside>
<main id="main">
<div id="log"></div>
@@ -81,10 +85,15 @@
</main>
<script>
const $ = id => document.getElementById(id);
const cfg = ['base','key','model','sys','temp','max'];
const cfg = ['base','key','model','sys','temp','max','ttsUrl','ttsVoice'];
cfg.forEach(k => { const v = localStorage.getItem('gc_'+k); if (v !== null) $(k).value = v; });
if (!$('base').value) $('base').value = 'http://10.250.50.70:4000/v1';
if (!$('ttsUrl').value) $('ttsUrl').value = 'http://10.100.79.3:8299/tts';
if (!$('ttsVoice').value) $('ttsVoice').value = 'baddy';
cfg.forEach(k => $(k).addEventListener('input', e => localStorage.setItem('gc_'+k, e.target.value)));
// 🔊 auto-play toggle is a checkbox (persist .checked)
$('ttsOn').checked = localStorage.getItem('gc_ttsOn') === '1';
$('ttsOn').addEventListener('change', e => localStorage.setItem('gc_ttsOn', e.target.checked ? '1' : '0'));
let history = []; // [{role, content}] content = string | multimodal array
let attached = null; // { url:dataURL, name } staged for the NEXT message
@@ -135,6 +144,35 @@ function addMsg(who, cls){
}
function showErr(text){ const m = addMsg('error','err'); m.body.className = 'err'; m.body.textContent = text; }
// --- mOrpheus TTS: auto-voice quoted text from a completed assistant reply ---
let ttsAudio = null; // current clip, so a new reply interrupts the previous one
function extractQuotes(text){
const re = /“([^”]+)”|"([^"]+)"/g; // typographic "…" or straight "…"
const out = []; let m;
while ((m = re.exec(text)) !== null){ const q = (m[1] || m[2] || '').trim(); if (q) out.push(q); }
return out;
}
async function speakQuotes(text, msg){
if (!$('ttsOn').checked) return;
const quotes = extractQuotes(text);
if (!quotes.length) return;
const tag = document.createElement('span'); tag.textContent = ' 🔊';
tag.style.cursor = 'pointer'; tag.title = 'mOrpheus'; msg.wrap.querySelector('.who').append(tag);
try {
const res = await fetch($('ttsUrl').value.trim(), {
method:'POST', headers:{ 'Content-Type':'application/json' },
body: JSON.stringify({ text: quotes.join(' '), voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 1200 })
});
if (!res.ok){ tag.textContent = ' 🔇' + res.status; return; }
const url = URL.createObjectURL(await res.blob());
if (ttsAudio) ttsAudio.pause();
ttsAudio = new Audio(url);
tag.onclick = () => ttsAudio.play(); // click 🔊 to replay
ttsAudio.onended = () => tag.textContent = ' 🔈';
ttsAudio.play();
} catch (e){ tag.textContent = ' 🔇'; }
}
async function send(){
const text = $('inp').value.trim();
if (!text && !attached) return;
@@ -196,6 +234,7 @@ async function send(){
}
}
history.push({ role:'assistant', content });
speakQuotes(content, out);
} catch (e){
showErr(String(e)); out.wrap.remove();
} finally {
stacks/mOrpheus/tts/app.py
+5
View File
@@ -16,6 +16,7 @@ import numpy as np, torch, soundfile as sf, requests
from scipy.signal import resample_poly
from fastapi import FastAPI, HTTPException
from fastapi.responses import Response
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from transformers import AutoTokenizer
from snac import SNAC
@@ -33,6 +34,10 @@ AUDIO_BASE, SOS, EOS_SP, SOH, SOA, EOT, BOS = 128266, 128257, 128258, 128259, 12
tok = AutoTokenizer.from_pretrained(MODEL_DIR)
snac_model = SNAC.from_pretrained(SNAC_DIR).to(SNAC_DEVICE).eval()
app = FastAPI(title="mOrpheus TTS", version="0.1.0")
# INTERNAL RESEARCH: browser callers (e.g. gateway-chat on ana-docker:8091) fetch this
# cross-origin; allow all origins on this internal-only endpoint.
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"],
allow_headers=["*"], expose_headers=["X-Audio-Seconds"])
class TTSReq(BaseModel):
tools/gateway-chat.html
+41 -2
View File
@@ -67,7 +67,11 @@
<div><label>Temperature</label><input id="temp" type="number" step="0.05" value="1"></div>
<div><label>Max tokens</label><input id="max" type="number" step="1" value="1024"></div>
</div>
<div class="hint">Settings persist locally. No <code>tools</code> field is ever sent. Reasoning models show their <code>reasoning_content</code> above the answer.</div>
<label style="margin-top:.5rem">🔊 mOrpheus voice (quoted text)</label>
<label class="hint" style="display:flex;gap:.45rem;align-items:center;margin:.15rem 0"><input id="ttsOn" type="checkbox" style="width:auto"> Auto-play quoted dialogue</label>
<input id="ttsUrl" placeholder="http://10.100.79.3:8299/tts">
<input id="ttsVoice" placeholder="baddy">
<div class="hint">Settings persist locally. No <code>tools</code> field is ever sent. Reasoning models show their <code>reasoning_content</code> above the answer. With 🔊 on, quoted text in each reply auto-plays via mOrpheus (irv-ml1, internal).</div>
</aside>
<main id="main">
<div id="log"></div>
@@ -81,10 +85,15 @@
</main>
<script>
const $ = id => document.getElementById(id);
const cfg = ['base','key','model','sys','temp','max'];
const cfg = ['base','key','model','sys','temp','max','ttsUrl','ttsVoice'];
cfg.forEach(k => { const v = localStorage.getItem('gc_'+k); if (v !== null) $(k).value = v; });
if (!$('base').value) $('base').value = 'http://10.250.50.70:4000/v1';
if (!$('ttsUrl').value) $('ttsUrl').value = 'http://10.100.79.3:8299/tts';
if (!$('ttsVoice').value) $('ttsVoice').value = 'baddy';
cfg.forEach(k => $(k).addEventListener('input', e => localStorage.setItem('gc_'+k, e.target.value)));
// 🔊 auto-play toggle is a checkbox (persist .checked)
$('ttsOn').checked = localStorage.getItem('gc_ttsOn') === '1';
$('ttsOn').addEventListener('change', e => localStorage.setItem('gc_ttsOn', e.target.checked ? '1' : '0'));
let history = []; // [{role, content}] content = string | multimodal array
let attached = null; // { url:dataURL, name } staged for the NEXT message
@@ -135,6 +144,35 @@ function addMsg(who, cls){
}
function showErr(text){ const m = addMsg('error','err'); m.body.className = 'err'; m.body.textContent = text; }
// --- mOrpheus TTS: auto-voice quoted text from a completed assistant reply ---
let ttsAudio = null; // current clip, so a new reply interrupts the previous one
function extractQuotes(text){
const re = /“([^”]+)”|"([^"]+)"/g; // typographic "…" or straight "…"
const out = []; let m;
while ((m = re.exec(text)) !== null){ const q = (m[1] || m[2] || '').trim(); if (q) out.push(q); }
return out;
}
async function speakQuotes(text, msg){
if (!$('ttsOn').checked) return;
const quotes = extractQuotes(text);
if (!quotes.length) return;
const tag = document.createElement('span'); tag.textContent = ' 🔊';
tag.style.cursor = 'pointer'; tag.title = 'mOrpheus'; msg.wrap.querySelector('.who').append(tag);
try {
const res = await fetch($('ttsUrl').value.trim(), {
method:'POST', headers:{ 'Content-Type':'application/json' },
body: JSON.stringify({ text: quotes.join(' '), voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 1200 })
});
if (!res.ok){ tag.textContent = ' 🔇' + res.status; return; }
const url = URL.createObjectURL(await res.blob());
if (ttsAudio) ttsAudio.pause();
ttsAudio = new Audio(url);
tag.onclick = () => ttsAudio.play(); // click 🔊 to replay
ttsAudio.onended = () => tag.textContent = ' 🔈';
ttsAudio.play();
} catch (e){ tag.textContent = ' 🔇'; }
}
async function send(){
const text = $('inp').value.trim();
if (!text && !attached) return;
@@ -196,6 +234,7 @@ async function send(){
}
}
history.push({ role:'assistant', content });
speakQuotes(content, out);
} catch (e){
showErr(String(e)); out.wrap.remove();
} finally {