feat(gateway-chat): auto-voice quoted dialogue via mOrpheus TTS
Gateway-chat now auto-plays quoted text from each assistant reply through the mOrpheus TTS endpoint. Sidebar gains a 🔊 toggle + endpoint/voice fields (persist in localStorage, prefilled to irv-ml1:8299 / baddy). On reply-complete, straight and typographic double quotes are extracted, joined, POSTed to /tts, and the returned WAV plays (click 🔊 to replay; a new reply interrupts the prior clip). Requires CORS on the wrapper (page served from ana-docker:8091 fetches irv-ml1:8299 cross-origin) — added CORSMiddleware(allow_origins=[*]) to the mOrpheus tts app (internal- only endpoint). Verified end-to-end: preflight + POST return ACAO=*, valid 24kHz WAV. Deployed: tts container rebuilt/recreated on irv-ml1; page pushed to ana-docker conf (bind-mounted, live on next request).
This commit is contained in:
@@ -67,7 +67,11 @@
|
|||||||
<div><label>Temperature</label><input id="temp" type="number" step="0.05" value="1"></div>
|
<div><label>Temperature</label><input id="temp" type="number" step="0.05" value="1"></div>
|
||||||
<div><label>Max tokens</label><input id="max" type="number" step="1" value="1024"></div>
|
<div><label>Max tokens</label><input id="max" type="number" step="1" value="1024"></div>
|
||||||
</div>
|
</div>
|
||||||
<div class="hint">Settings persist locally. No <code>tools</code> field is ever sent. Reasoning models show their <code>reasoning_content</code> above the answer.</div>
|
<label style="margin-top:.5rem">🔊 mOrpheus voice (quoted text)</label>
|
||||||
|
<label class="hint" style="display:flex;gap:.45rem;align-items:center;margin:.15rem 0"><input id="ttsOn" type="checkbox" style="width:auto"> Auto-play quoted dialogue</label>
|
||||||
|
<input id="ttsUrl" placeholder="http://10.100.79.3:8299/tts">
|
||||||
|
<input id="ttsVoice" placeholder="baddy">
|
||||||
|
<div class="hint">Settings persist locally. No <code>tools</code> field is ever sent. Reasoning models show their <code>reasoning_content</code> above the answer. With 🔊 on, quoted text in each reply auto-plays via mOrpheus (irv-ml1, internal).</div>
|
||||||
</aside>
|
</aside>
|
||||||
<main id="main">
|
<main id="main">
|
||||||
<div id="log"></div>
|
<div id="log"></div>
|
||||||
@@ -81,10 +85,15 @@
|
|||||||
</main>
|
</main>
|
||||||
<script>
|
<script>
|
||||||
const $ = id => document.getElementById(id);
|
const $ = id => document.getElementById(id);
|
||||||
const cfg = ['base','key','model','sys','temp','max'];
|
const cfg = ['base','key','model','sys','temp','max','ttsUrl','ttsVoice'];
|
||||||
cfg.forEach(k => { const v = localStorage.getItem('gc_'+k); if (v !== null) $(k).value = v; });
|
cfg.forEach(k => { const v = localStorage.getItem('gc_'+k); if (v !== null) $(k).value = v; });
|
||||||
if (!$('base').value) $('base').value = 'http://10.250.50.70:4000/v1';
|
if (!$('base').value) $('base').value = 'http://10.250.50.70:4000/v1';
|
||||||
|
if (!$('ttsUrl').value) $('ttsUrl').value = 'http://10.100.79.3:8299/tts';
|
||||||
|
if (!$('ttsVoice').value) $('ttsVoice').value = 'baddy';
|
||||||
cfg.forEach(k => $(k).addEventListener('input', e => localStorage.setItem('gc_'+k, e.target.value)));
|
cfg.forEach(k => $(k).addEventListener('input', e => localStorage.setItem('gc_'+k, e.target.value)));
|
||||||
|
// 🔊 auto-play toggle is a checkbox (persist .checked)
|
||||||
|
$('ttsOn').checked = localStorage.getItem('gc_ttsOn') === '1';
|
||||||
|
$('ttsOn').addEventListener('change', e => localStorage.setItem('gc_ttsOn', e.target.checked ? '1' : '0'));
|
||||||
|
|
||||||
let history = []; // [{role, content}] content = string | multimodal array
|
let history = []; // [{role, content}] content = string | multimodal array
|
||||||
let attached = null; // { url:dataURL, name } staged for the NEXT message
|
let attached = null; // { url:dataURL, name } staged for the NEXT message
|
||||||
@@ -135,6 +144,35 @@ function addMsg(who, cls){
|
|||||||
}
|
}
|
||||||
function showErr(text){ const m = addMsg('error','err'); m.body.className = 'err'; m.body.textContent = text; }
|
function showErr(text){ const m = addMsg('error','err'); m.body.className = 'err'; m.body.textContent = text; }
|
||||||
|
|
||||||
|
// --- mOrpheus TTS: auto-voice quoted text from a completed assistant reply ---
|
||||||
|
let ttsAudio = null; // current clip, so a new reply interrupts the previous one
|
||||||
|
function extractQuotes(text){
|
||||||
|
const re = /“([^”]+)”|"([^"]+)"/g; // typographic "…" or straight "…"
|
||||||
|
const out = []; let m;
|
||||||
|
while ((m = re.exec(text)) !== null){ const q = (m[1] || m[2] || '').trim(); if (q) out.push(q); }
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
async function speakQuotes(text, msg){
|
||||||
|
if (!$('ttsOn').checked) return;
|
||||||
|
const quotes = extractQuotes(text);
|
||||||
|
if (!quotes.length) return;
|
||||||
|
const tag = document.createElement('span'); tag.textContent = ' 🔊';
|
||||||
|
tag.style.cursor = 'pointer'; tag.title = 'mOrpheus'; msg.wrap.querySelector('.who').append(tag);
|
||||||
|
try {
|
||||||
|
const res = await fetch($('ttsUrl').value.trim(), {
|
||||||
|
method:'POST', headers:{ 'Content-Type':'application/json' },
|
||||||
|
body: JSON.stringify({ text: quotes.join(' '), voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 1200 })
|
||||||
|
});
|
||||||
|
if (!res.ok){ tag.textContent = ' 🔇' + res.status; return; }
|
||||||
|
const url = URL.createObjectURL(await res.blob());
|
||||||
|
if (ttsAudio) ttsAudio.pause();
|
||||||
|
ttsAudio = new Audio(url);
|
||||||
|
tag.onclick = () => ttsAudio.play(); // click 🔊 to replay
|
||||||
|
ttsAudio.onended = () => tag.textContent = ' 🔈';
|
||||||
|
ttsAudio.play();
|
||||||
|
} catch (e){ tag.textContent = ' 🔇'; }
|
||||||
|
}
|
||||||
|
|
||||||
async function send(){
|
async function send(){
|
||||||
const text = $('inp').value.trim();
|
const text = $('inp').value.trim();
|
||||||
if (!text && !attached) return;
|
if (!text && !attached) return;
|
||||||
@@ -196,6 +234,7 @@ async function send(){
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
history.push({ role:'assistant', content });
|
history.push({ role:'assistant', content });
|
||||||
|
speakQuotes(content, out);
|
||||||
} catch (e){
|
} catch (e){
|
||||||
showErr(String(e)); out.wrap.remove();
|
showErr(String(e)); out.wrap.remove();
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import numpy as np, torch, soundfile as sf, requests
|
|||||||
from scipy.signal import resample_poly
|
from scipy.signal import resample_poly
|
||||||
from fastapi import FastAPI, HTTPException
|
from fastapi import FastAPI, HTTPException
|
||||||
from fastapi.responses import Response
|
from fastapi.responses import Response
|
||||||
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from transformers import AutoTokenizer
|
from transformers import AutoTokenizer
|
||||||
from snac import SNAC
|
from snac import SNAC
|
||||||
@@ -33,6 +34,10 @@ AUDIO_BASE, SOS, EOS_SP, SOH, SOA, EOT, BOS = 128266, 128257, 128258, 128259, 12
|
|||||||
tok = AutoTokenizer.from_pretrained(MODEL_DIR)
|
tok = AutoTokenizer.from_pretrained(MODEL_DIR)
|
||||||
snac_model = SNAC.from_pretrained(SNAC_DIR).to(SNAC_DEVICE).eval()
|
snac_model = SNAC.from_pretrained(SNAC_DIR).to(SNAC_DEVICE).eval()
|
||||||
app = FastAPI(title="mOrpheus TTS", version="0.1.0")
|
app = FastAPI(title="mOrpheus TTS", version="0.1.0")
|
||||||
|
# INTERNAL RESEARCH: browser callers (e.g. gateway-chat on ana-docker:8091) fetch this
|
||||||
|
# cross-origin; allow all origins on this internal-only endpoint.
|
||||||
|
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"],
|
||||||
|
allow_headers=["*"], expose_headers=["X-Audio-Seconds"])
|
||||||
|
|
||||||
|
|
||||||
class TTSReq(BaseModel):
|
class TTSReq(BaseModel):
|
||||||
|
|||||||
+41
-2
@@ -67,7 +67,11 @@
|
|||||||
<div><label>Temperature</label><input id="temp" type="number" step="0.05" value="1"></div>
|
<div><label>Temperature</label><input id="temp" type="number" step="0.05" value="1"></div>
|
||||||
<div><label>Max tokens</label><input id="max" type="number" step="1" value="1024"></div>
|
<div><label>Max tokens</label><input id="max" type="number" step="1" value="1024"></div>
|
||||||
</div>
|
</div>
|
||||||
<div class="hint">Settings persist locally. No <code>tools</code> field is ever sent. Reasoning models show their <code>reasoning_content</code> above the answer.</div>
|
<label style="margin-top:.5rem">🔊 mOrpheus voice (quoted text)</label>
|
||||||
|
<label class="hint" style="display:flex;gap:.45rem;align-items:center;margin:.15rem 0"><input id="ttsOn" type="checkbox" style="width:auto"> Auto-play quoted dialogue</label>
|
||||||
|
<input id="ttsUrl" placeholder="http://10.100.79.3:8299/tts">
|
||||||
|
<input id="ttsVoice" placeholder="baddy">
|
||||||
|
<div class="hint">Settings persist locally. No <code>tools</code> field is ever sent. Reasoning models show their <code>reasoning_content</code> above the answer. With 🔊 on, quoted text in each reply auto-plays via mOrpheus (irv-ml1, internal).</div>
|
||||||
</aside>
|
</aside>
|
||||||
<main id="main">
|
<main id="main">
|
||||||
<div id="log"></div>
|
<div id="log"></div>
|
||||||
@@ -81,10 +85,15 @@
|
|||||||
</main>
|
</main>
|
||||||
<script>
|
<script>
|
||||||
const $ = id => document.getElementById(id);
|
const $ = id => document.getElementById(id);
|
||||||
const cfg = ['base','key','model','sys','temp','max'];
|
const cfg = ['base','key','model','sys','temp','max','ttsUrl','ttsVoice'];
|
||||||
cfg.forEach(k => { const v = localStorage.getItem('gc_'+k); if (v !== null) $(k).value = v; });
|
cfg.forEach(k => { const v = localStorage.getItem('gc_'+k); if (v !== null) $(k).value = v; });
|
||||||
if (!$('base').value) $('base').value = 'http://10.250.50.70:4000/v1';
|
if (!$('base').value) $('base').value = 'http://10.250.50.70:4000/v1';
|
||||||
|
if (!$('ttsUrl').value) $('ttsUrl').value = 'http://10.100.79.3:8299/tts';
|
||||||
|
if (!$('ttsVoice').value) $('ttsVoice').value = 'baddy';
|
||||||
cfg.forEach(k => $(k).addEventListener('input', e => localStorage.setItem('gc_'+k, e.target.value)));
|
cfg.forEach(k => $(k).addEventListener('input', e => localStorage.setItem('gc_'+k, e.target.value)));
|
||||||
|
// 🔊 auto-play toggle is a checkbox (persist .checked)
|
||||||
|
$('ttsOn').checked = localStorage.getItem('gc_ttsOn') === '1';
|
||||||
|
$('ttsOn').addEventListener('change', e => localStorage.setItem('gc_ttsOn', e.target.checked ? '1' : '0'));
|
||||||
|
|
||||||
let history = []; // [{role, content}] content = string | multimodal array
|
let history = []; // [{role, content}] content = string | multimodal array
|
||||||
let attached = null; // { url:dataURL, name } staged for the NEXT message
|
let attached = null; // { url:dataURL, name } staged for the NEXT message
|
||||||
@@ -135,6 +144,35 @@ function addMsg(who, cls){
|
|||||||
}
|
}
|
||||||
function showErr(text){ const m = addMsg('error','err'); m.body.className = 'err'; m.body.textContent = text; }
|
function showErr(text){ const m = addMsg('error','err'); m.body.className = 'err'; m.body.textContent = text; }
|
||||||
|
|
||||||
|
// --- mOrpheus TTS: auto-voice quoted text from a completed assistant reply ---
|
||||||
|
let ttsAudio = null; // current clip, so a new reply interrupts the previous one
|
||||||
|
function extractQuotes(text){
|
||||||
|
const re = /“([^”]+)”|"([^"]+)"/g; // typographic "…" or straight "…"
|
||||||
|
const out = []; let m;
|
||||||
|
while ((m = re.exec(text)) !== null){ const q = (m[1] || m[2] || '').trim(); if (q) out.push(q); }
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
async function speakQuotes(text, msg){
|
||||||
|
if (!$('ttsOn').checked) return;
|
||||||
|
const quotes = extractQuotes(text);
|
||||||
|
if (!quotes.length) return;
|
||||||
|
const tag = document.createElement('span'); tag.textContent = ' 🔊';
|
||||||
|
tag.style.cursor = 'pointer'; tag.title = 'mOrpheus'; msg.wrap.querySelector('.who').append(tag);
|
||||||
|
try {
|
||||||
|
const res = await fetch($('ttsUrl').value.trim(), {
|
||||||
|
method:'POST', headers:{ 'Content-Type':'application/json' },
|
||||||
|
body: JSON.stringify({ text: quotes.join(' '), voice: ($('ttsVoice').value.trim() || 'baddy'), max_tokens: 1200 })
|
||||||
|
});
|
||||||
|
if (!res.ok){ tag.textContent = ' 🔇' + res.status; return; }
|
||||||
|
const url = URL.createObjectURL(await res.blob());
|
||||||
|
if (ttsAudio) ttsAudio.pause();
|
||||||
|
ttsAudio = new Audio(url);
|
||||||
|
tag.onclick = () => ttsAudio.play(); // click 🔊 to replay
|
||||||
|
ttsAudio.onended = () => tag.textContent = ' 🔈';
|
||||||
|
ttsAudio.play();
|
||||||
|
} catch (e){ tag.textContent = ' 🔇'; }
|
||||||
|
}
|
||||||
|
|
||||||
async function send(){
|
async function send(){
|
||||||
const text = $('inp').value.trim();
|
const text = $('inp').value.trim();
|
||||||
if (!text && !attached) return;
|
if (!text && !attached) return;
|
||||||
@@ -196,6 +234,7 @@ async function send(){
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
history.push({ role:'assistant', content });
|
history.push({ role:'assistant', content });
|
||||||
|
speakQuotes(content, out);
|
||||||
} catch (e){
|
} catch (e){
|
||||||
showErr(String(e)); out.wrap.remove();
|
showErr(String(e)); out.wrap.remove();
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
Reference in New Issue
Block a user