vh
6a90a70ad8
Point the in-page TTS at the zonos-gateway wrapper on irv-ml1:8890 (direct, so streaming isn't buffered by LiteLLM): OpenAI-shape body (model: ext-tts, input, voice), Cora default voice, float32@44.1kHz PCM decode. Quotes are joined into a single stream call (prosody — no per-sentence chunking). Migration regex rewrites stale saved endpoints (:8299/:8210, /tts[/stream], :4000) to the new one.
299 lines
16 KiB
HTML
299 lines
16 KiB
HTML
<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<title>Gateway Chat — light LiteLLM playground</title>
|
|
<!--
|
|
Single-file chat playground for the LiteLLM gateway (10.250.50.70:4000).
|
|
Zero dependencies, zero backend: open it in a browser (file:// is fine) or
|
|
serve statically. The gateway sends permissive CORS, so the browser talks to
|
|
it directly. It deliberately NEVER sends a `tools` field, sidestepping the
|
|
vLLM "tools must not be an empty array" bug that breaks the LiteLLM UI
|
|
playground for vLLM-backed models. Settings persist in localStorage.
|
|
|
|
Built for SMOKING new models: the Model field auto-discovers every model on
|
|
the gateway (the ↻ control pulls /v1/models — new models just appear), and 📎
|
|
attaches an image so you can smoke multimodal/vision models (Qwopus,
|
|
image-judge, …) too. Image is sent as a base64 data: URL in image_url content.
|
|
-->
|
|
<style>
|
|
:root{--bg:#0f1115;--panel:#171a21;--ink:#e6e9ef;--muted:#8b93a7;--acc:#5b8cff;--line:#262b36;--user:#1f6feb22;--warn:#ff6b6b}
|
|
*{box-sizing:border-box}
|
|
body{margin:0;font:14px/1.55 system-ui,sans-serif;background:var(--bg);color:var(--ink);display:flex;height:100vh}
|
|
#side{width:330px;min-width:330px;background:var(--panel);border-right:1px solid var(--line);padding:14px;overflow:auto}
|
|
#main{flex:1;display:flex;flex-direction:column;min-width:0}
|
|
h1{font-size:13px;text-transform:uppercase;letter-spacing:.08em;color:var(--muted);margin:0 0 12px}
|
|
label{display:block;font-size:11px;text-transform:uppercase;letter-spacing:.05em;color:var(--muted);margin:10px 0 4px}
|
|
input,textarea,select{width:100%;background:#0c0e13;color:var(--ink);border:1px solid var(--line);border-radius:6px;padding:7px 8px;font:inherit}
|
|
textarea{resize:vertical}
|
|
#sys{min-height:130px;font-family:ui-monospace,monospace;font-size:12px}
|
|
.row{display:flex;gap:8px}.row>*{flex:1}
|
|
#log{flex:1;overflow:auto;padding:18px;display:flex;flex-direction:column;gap:14px}
|
|
.msg{max-width:840px}
|
|
.msg .who{font-size:11px;text-transform:uppercase;letter-spacing:.05em;color:var(--muted);margin-bottom:3px}
|
|
.msg .body{white-space:pre-wrap;word-wrap:break-word}
|
|
.msg.user .body{background:var(--user);border:1px solid #1f6feb55;border-radius:8px;padding:8px 10px}
|
|
.msg .reason{white-space:pre-wrap;color:var(--muted);font-size:12.5px;border-left:2px solid var(--line);padding-left:10px;margin-bottom:6px}
|
|
#composer{border-top:1px solid var(--line);padding:12px 18px;display:flex;gap:10px;background:var(--panel)}
|
|
#inp{flex:1;min-height:42px;max-height:220px}
|
|
button{background:var(--acc);color:#fff;border:0;border-radius:6px;padding:0 16px;font:inherit;font-weight:600;cursor:pointer}
|
|
button.ghost{background:transparent;color:var(--muted);border:1px solid var(--line)}
|
|
button:disabled{opacity:.5;cursor:default}
|
|
.err{color:var(--warn);font-family:ui-monospace,monospace;font-size:12px;white-space:pre-wrap}
|
|
.hint{font-size:11px;color:var(--muted);margin-top:8px}
|
|
code{font-family:ui-monospace,monospace}
|
|
#refresh{cursor:pointer;color:var(--acc);font-size:10px;letter-spacing:0}
|
|
#composer{align-items:flex-end}
|
|
#attach{flex:0 0 auto;display:flex;align-items:center;justify-content:center;padding:0 12px;height:42px;font-size:18px}
|
|
#attached{display:none;align-items:center;gap:8px;font-size:11px;color:var(--muted);padding:6px 18px 0;background:var(--panel)}
|
|
#attached img{height:30px;border-radius:4px;border:1px solid var(--line)}
|
|
.msg .thumb{max-width:220px;max-height:220px;border-radius:6px;border:1px solid var(--line);margin-bottom:6px;display:block}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<aside id="side">
|
|
<h1>Gateway Chat</h1>
|
|
<label>Base URL</label>
|
|
<input id="base" placeholder="http://10.250.50.70:4000/v1">
|
|
<label>API key</label>
|
|
<input id="key" type="password" placeholder="sk-…">
|
|
<label>Model <span id="refresh" title="Reload model list from /v1/models">↻ load</span></label>
|
|
<input id="model" list="models" placeholder="(pick or type — ↻ loads all gateway models)">
|
|
<datalist id="models"></datalist>
|
|
<label>System prompt</label>
|
|
<textarea id="sys" placeholder="You are a helpful assistant."></textarea>
|
|
<div class="row">
|
|
<div><label>Temperature</label><input id="temp" type="number" step="0.05" value="1"></div>
|
|
<div><label>Max tokens</label><input id="max" type="number" step="1" value="1024"></div>
|
|
</div>
|
|
<label style="margin-top:.5rem">🔊 ext-tts voice (quoted text)</label>
|
|
<label class="hint" style="display:flex;gap:.45rem;align-items:center;margin:.15rem 0"><input id="ttsOn" type="checkbox" style="width:auto"> Auto-play quoted dialogue</label>
|
|
<input id="ttsUrl" placeholder="http://10.100.79.3:8890/v1/audio/speech">
|
|
<input id="ttsVoice" placeholder="Cora">
|
|
<div class="hint">Settings persist locally. No <code>tools</code> field is ever sent. Reasoning models show their <code>reasoning_content</code> above the answer. With 🔊 on, quoted text in each reply auto-plays via <code>ext-tts</code> (Zonos, streamed direct from zonos-gateway).</div>
|
|
</aside>
|
|
<main id="main">
|
|
<div id="log"></div>
|
|
<div id="attached"></div>
|
|
<div id="composer">
|
|
<label id="attach" class="ghost" title="Attach an image (smoke vision models)">📎<input id="img" type="file" accept="image/*" hidden></label>
|
|
<textarea id="inp" placeholder="Message… (Enter to send · Shift+Enter for newline)"></textarea>
|
|
<button id="send">Send</button>
|
|
<button id="reset" class="ghost" title="Clear conversation">Reset</button>
|
|
</div>
|
|
</main>
|
|
<script>
|
|
const $ = id => document.getElementById(id);
|
|
const cfg = ['base','key','model','sys','temp','max','ttsUrl','ttsVoice'];
|
|
cfg.forEach(k => { const v = localStorage.getItem('gc_'+k); if (v !== null) $(k).value = v; });
|
|
if (!$('base').value) $('base').value = 'http://10.250.50.70:4000/v1';
|
|
// ext-tts default = zonos-gateway direct (streams; bypasses LiteLLM audio buffering). Migrate any stale
|
|
// mOrpheus URL (:8299/:8210//tts) OR the LiteLLM URL (:4000) to the direct gateway, persisting so it sticks.
|
|
if (!$('ttsUrl').value || /:(8299|8210)|\/tts(\/stream)?$|10\.250\.50\.70:4000/.test($('ttsUrl').value)){
|
|
$('ttsUrl').value = 'http://10.100.79.3:8890/v1/audio/speech'; localStorage.setItem('gc_ttsUrl', $('ttsUrl').value);
|
|
}
|
|
if (!$('ttsVoice').value || $('ttsVoice').value.trim() === 'baddy'){
|
|
$('ttsVoice').value = 'Cora'; localStorage.setItem('gc_ttsVoice', $('ttsVoice').value);
|
|
}
|
|
cfg.forEach(k => $(k).addEventListener('input', e => localStorage.setItem('gc_'+k, e.target.value)));
|
|
// 🔊 auto-play toggle is a checkbox (persist .checked)
|
|
$('ttsOn').checked = localStorage.getItem('gc_ttsOn') === '1';
|
|
$('ttsOn').addEventListener('change', e => localStorage.setItem('gc_ttsOn', e.target.checked ? '1' : '0'));
|
|
|
|
let history = []; // [{role, content}] content = string | multimodal array
|
|
let attached = null; // { url:dataURL, name } staged for the NEXT message
|
|
const log = $('log');
|
|
|
|
// --- model auto-discovery: fill the datalist from /v1/models (new models just appear) ---
|
|
async function loadModels(){
|
|
try {
|
|
const res = await fetch($('base').value.replace(/\/+$/,'') + '/models',
|
|
{ headers:{ 'Authorization':'Bearer ' + $('key').value } });
|
|
if (!res.ok){ $('refresh').textContent = '↻ ' + res.status; return; }
|
|
const ids = ((await res.json()).data || []).map(m => m.id).sort();
|
|
const dl = $('models'); dl.innerHTML = '';
|
|
for (const id of ids){ const o = document.createElement('option'); o.value = id; dl.append(o); }
|
|
$('refresh').textContent = '↻ ' + ids.length;
|
|
} catch (e){ $('refresh').textContent = '↻ offline'; }
|
|
}
|
|
$('refresh').addEventListener('click', loadModels);
|
|
$('base').addEventListener('change', loadModels);
|
|
$('key').addEventListener('change', loadModels);
|
|
loadModels();
|
|
|
|
// --- image attach (vision smoke): stage a base64 data: URL for the next send ---
|
|
$('img').addEventListener('change', e => {
|
|
const f = e.target.files[0]; if (!f) return;
|
|
const r = new FileReader();
|
|
r.onload = () => { attached = { url:r.result, name:f.name }; showAttached(); };
|
|
r.readAsDataURL(f);
|
|
});
|
|
function showAttached(){
|
|
const a = $('attached');
|
|
if (!attached){ a.style.display = 'none'; a.innerHTML = ''; return; }
|
|
a.style.display = 'flex'; a.innerHTML = '';
|
|
const im = document.createElement('img'); im.src = attached.url;
|
|
const nm = document.createElement('span'); nm.textContent = attached.name;
|
|
const x = document.createElement('span'); x.textContent = '✕ remove'; x.style.cursor = 'pointer'; x.style.color = 'var(--warn)';
|
|
x.onclick = () => { attached = null; $('img').value = ''; showAttached(); };
|
|
a.append(im, nm, x);
|
|
}
|
|
|
|
function addMsg(who, cls){
|
|
const wrap = document.createElement('div'); wrap.className = 'msg ' + cls;
|
|
const h = document.createElement('div'); h.className = 'who'; h.textContent = who;
|
|
const r = document.createElement('div'); r.className = 'reason'; r.style.display = 'none';
|
|
const b = document.createElement('div'); b.className = 'body';
|
|
wrap.append(h, r, b); log.append(wrap); log.scrollTop = log.scrollHeight;
|
|
return { wrap, reason:r, body:b };
|
|
}
|
|
function showErr(text){ const m = addMsg('error','err'); m.body.className = 'err'; m.body.textContent = text; }
|
|
|
|
// --- mOrpheus TTS: streaming-decode auto-voice of quoted text (Web Audio, low TTFA) ---
|
|
let audioCtx = null; // shared AudioContext
|
|
let ttsGen = 0; // generation counter — a newer reply's stream supersedes older ones
|
|
let speechHead = 0; // shared playback clock — sentences queue onto it back-to-back
|
|
function primeAudio(){ // browsers suspend the AudioContext until a user gesture; resume on ANY
|
|
if (!audioCtx) audioCtx = new (window.AudioContext || window.webkitAudioContext)();
|
|
if (audioCtx.state === 'suspended') audioCtx.resume(); // interaction so async playback isn't silent
|
|
}
|
|
document.addEventListener('click', primeAudio, true);
|
|
document.addEventListener('keydown', primeAudio, true);
|
|
function extractQuotes(text){
|
|
const re = /“([^”]+)”|"([^"]+)"/g; // typographic "…" or straight "…"
|
|
const out = []; let m;
|
|
while ((m = re.exec(text)) !== null){ const q = (m[1] || m[2] || '').trim(); if (q) out.push(q); }
|
|
return out;
|
|
}
|
|
// stream ONE quoted section (whole, so intonation/prosody across it is preserved) and queue its
|
|
// PCM onto the shared speechHead clock; resolves when its audio is fully received (it plays on
|
|
// while the NEXT quote starts generating). Chunk by QUOTE, not sentence — per-sentence lost prosody.
|
|
async function streamOne(sentence, mine){
|
|
// ext-tts via zonos-gateway (direct, streams — bypasses LiteLLM's audio buffering): ttsUrl is the full
|
|
// OpenAI /v1/audio/speech endpoint. The gateway relays Zonos's raw float32 PCM @ 44.1kHz — decode as
|
|
// little-endian float32 (NOT int16) at 44100 Hz. The Bearer header is sent but the gateway ignores it.
|
|
const url = $('ttsUrl').value.trim().replace(/\/+$/,'');
|
|
const res = await fetch(url, {
|
|
method:'POST',
|
|
headers:{ 'Content-Type':'application/json', 'Authorization':'Bearer ' + $('key').value },
|
|
body: JSON.stringify({ model: 'ext-tts', input: sentence, voice: ($('ttsVoice').value.trim() || 'Cora') })
|
|
});
|
|
if (!res.ok || !res.body) return false;
|
|
const reader = res.body.getReader();
|
|
let leftover = new Uint8Array(0);
|
|
for(;;){
|
|
const { done, value } = await reader.read();
|
|
if (done || mine !== ttsGen) break; // stop if a newer reply took over
|
|
let bytes = new Uint8Array(leftover.length + value.length);
|
|
bytes.set(leftover); bytes.set(value, leftover.length);
|
|
const n = bytes.length - (bytes.length % 4); // whole float32 samples only (ext-tts PCM)
|
|
leftover = bytes.slice(n);
|
|
if (!n) continue;
|
|
const dv = new DataView(bytes.buffer, 0, n), N = n / 4, f32 = new Float32Array(N);
|
|
for (let i = 0; i < N; i++) f32[i] = dv.getFloat32(i * 4, true); // Zonos float32 PCM, little-endian
|
|
const abuf = audioCtx.createBuffer(1, N, 44100); abuf.getChannelData(0).set(f32);
|
|
const src = audioCtx.createBufferSource(); src.buffer = abuf; src.connect(audioCtx.destination);
|
|
if (speechHead < audioCtx.currentTime + 0.05) speechHead = audioCtx.currentTime + 0.05; // underrun catch-up
|
|
src.start(speechHead); speechHead += abuf.duration; // queue back-to-back on the shared clock
|
|
}
|
|
return true;
|
|
}
|
|
async function speakQuotes(text, msg){
|
|
if (!$('ttsOn').checked) return;
|
|
const quotes = extractQuotes(text);
|
|
if (!quotes.length) return;
|
|
// Streaming backend (zonos-gateway) → send ALL quoted dialogue as ONE streamed request;
|
|
// the stream itself is the incremental playback, so no per-quote chunking is needed.
|
|
const joined = quotes.join(' ');
|
|
const tag = document.createElement('span'); tag.textContent = ' 🔊';
|
|
tag.style.cursor = 'pointer'; tag.title = 'ext-tts — ' + quotes.length + ' quote(s), click to replay';
|
|
const play = async () => {
|
|
const mine = ++ttsGen; // supersede any in-flight playback
|
|
primeAudio(); if (audioCtx.state === 'suspended') await audioCtx.resume();
|
|
speechHead = audioCtx.currentTime + 0.12; // reset the shared clock for this reply
|
|
if (mine !== ttsGen) return;
|
|
if (!(await streamOne(joined, mine))){ tag.textContent = ' 🔇'; return; }
|
|
if (mine === ttsGen) tag.textContent = ' 🔈';
|
|
};
|
|
tag.onclick = () => play().catch(() => tag.textContent = ' 🔇');
|
|
msg.wrap.querySelector('.who').append(tag);
|
|
try { await play(); } catch (e){ tag.textContent = ' 🔇'; }
|
|
}
|
|
|
|
async function send(){
|
|
const text = $('inp').value.trim();
|
|
if (!text && !attached) return;
|
|
$('inp').value = '';
|
|
|
|
// multimodal array if an image is staged, else a plain string
|
|
let userContent;
|
|
if (attached){
|
|
userContent = [];
|
|
if (text) userContent.push({ type:'text', text });
|
|
userContent.push({ type:'image_url', image_url:{ url:attached.url } });
|
|
} else {
|
|
userContent = text;
|
|
}
|
|
history.push({ role:'user', content:userContent });
|
|
|
|
const u = addMsg('you','user');
|
|
if (attached){ const th = document.createElement('img'); th.className = 'thumb'; th.src = attached.url; u.body.append(th); }
|
|
u.body.append(document.createTextNode(text));
|
|
attached = null; $('img').value = ''; showAttached();
|
|
|
|
const messages = [];
|
|
const sys = $('sys').value.trim();
|
|
if (sys) messages.push({ role:'system', content:sys });
|
|
messages.push(...history);
|
|
|
|
const body = {
|
|
model: $('model').value.trim() || 'gen',
|
|
messages,
|
|
temperature: parseFloat($('temp').value),
|
|
max_tokens: parseInt($('max').value, 10),
|
|
stream: true
|
|
// intentionally NO `tools` — vLLM 400s on an empty array.
|
|
};
|
|
|
|
const out = addMsg(body.model, 'assistant');
|
|
let content = '', reasoning = '';
|
|
$('send').disabled = true;
|
|
try {
|
|
const res = await fetch($('base').value.replace(/\/+$/,'') + '/chat/completions', {
|
|
method:'POST',
|
|
headers:{ 'Content-Type':'application/json', 'Authorization':'Bearer ' + $('key').value },
|
|
body: JSON.stringify(body)
|
|
});
|
|
if (!res.ok){ showErr('HTTP ' + res.status + '\n' + await res.text()); out.wrap.remove(); return; }
|
|
const reader = res.body.getReader(); const dec = new TextDecoder(); let buf = '';
|
|
for(;;){
|
|
const { done, value } = await reader.read(); if (done) break;
|
|
buf += dec.decode(value, { stream:true });
|
|
const lines = buf.split('\n'); buf = lines.pop();
|
|
for (const line of lines){
|
|
const t = line.trim(); if (!t.startsWith('data:')) continue;
|
|
const payload = t.slice(5).trim(); if (payload === '[DONE]') continue;
|
|
let j; try { j = JSON.parse(payload); } catch { continue; }
|
|
const d = (j.choices && j.choices[0] && j.choices[0].delta) || {};
|
|
if (d.reasoning_content){ reasoning += d.reasoning_content; out.reason.style.display = 'block'; out.reason.textContent = reasoning; }
|
|
if (d.content){ content += d.content; out.body.textContent = content; }
|
|
log.scrollTop = log.scrollHeight;
|
|
}
|
|
}
|
|
history.push({ role:'assistant', content });
|
|
speakQuotes(content, out);
|
|
} catch (e){
|
|
showErr(String(e)); out.wrap.remove();
|
|
} finally {
|
|
$('send').disabled = false;
|
|
}
|
|
}
|
|
|
|
$('send').addEventListener('click', send);
|
|
$('inp').addEventListener('keydown', e => { if (e.key === 'Enter' && !e.shiftKey){ e.preventDefault(); send(); } });
|
|
$('reset').addEventListener('click', () => { history = []; log.innerHTML = ''; attached = null; $('img').value = ''; showAttached(); });
|
|
</script>
|
|
</body>
|
|
</html>
|