diff --git a/stacks/mOrpheus/.env.example b/stacks/mOrpheus/.env.example new file mode 100644 index 0000000..e41021e --- /dev/null +++ b/stacks/mOrpheus/.env.example @@ -0,0 +1,23 @@ +# mOrpheus stack tunables (irv-ml1). Copy to .env on the host; edit values there. + +# GPU pin (UUID). Default = the 3090 (the stabler audio-zoo GPU; A6000 is comfy's + spiky). +# irv-ml1: 3090 = GPU-12c15b3c-0827-0bab-b0cd-31aaa37a8e72 +# A6000 = GPU-9672f0d5-3caf-b15c-898a-cd9d653c654b +MORPHEUS_GPU_UUID=GPU-12c15b3c-0827-0bab-b0cd-31aaa37a8e72 + +# Weights (stage once; not in git). mOrpheus = LlamaForCausalLM 3.3B bf16 (6.6GB). +MORPHEUS_MODEL_DIR=/home/lkraven/morpheus/models/mOrpheus +SNAC_DIR=/home/lkraven/morpheus/models/snac_24khz + +# Engine. bf16 REQUIRED — FP8 destroys audio-token generation on this model (0 valid SNAC +# frames even at greedy). bf16 weights ~6.6GB; --enforce-eager (in compose) skips CUDA- +# graph capture so it fits the shared 3090. util 0.32 (~7.7GB of the 24GB card); raise if +# you free the 3090, lower only if you can (bf16 weights alone are 6.6GB => >=~0.30 floor). +MORPHEUS_GPU_UTIL=0.32 +MORPHEUS_MAX_LEN=4096 +MORPHEUS_MAX_SEQS=8 + +# TTS endpoint (host port). Public /tts + /docs live here. +MORPHEUS_TTS_PORT=8299 +MORPHEUS_DEFAULT_VOICE=baddy +MORPHEUS_VOICES=baddy diff --git a/stacks/mOrpheus/README.md b/stacks/mOrpheus/README.md new file mode 100644 index 0000000..153cf62 --- /dev/null +++ b/stacks/mOrpheus/README.md @@ -0,0 +1,54 @@ +# mOrpheus — uncensored Orpheus TTS (irv-ml1) + +Permanent serving stack for **`MrDragonFox/mOrpheus_3B-1Base_early_preview-v1-25000`** — an +uncensored Orpheus TTS finetune (Llama-3.2-3B LLM → SNAC 24 kHz audio). Trained speaker +**"baddy"**; supports **zero-shot voice cloning** from a reference clip. + +> **INTERNAL RESEARCH ONLY.** License is **CC-BY-NC-4.0** (non-commercial). Do **not** expose +> this endpoint externally or use it in any commercial-facing product. + +## Shape + +Two containers (see `compose.yaml`): + +| service | where | role | +|---|---|---| +| `vllm-morpheus` | GPU (3090), FP8 | serves the mOrpheus LLM; emits Orpheus audio tokens | +| `morpheus-tts` | CPU | SNAC-decodes tokens → 24 kHz WAV; the public `/tts` endpoint | + +**Real-time:** ~165 tok/s single-stream on the 3090 (FP8) ⇒ **RTF ≈ 0.50 (2× real-time)**, +measured. A ~4 s clip generates in ~2 s. (Whole-clip decode in v1; chunked streaming for +lower time-to-first-audio is a future enhancement.) + +## Endpoints (`http://10.100.79.3:8299`) + +- `POST /tts` → `audio/wav`. Body: `{"text": "...", "voice": "baddy", "temperature": 0.6, + "max_tokens": 1200, "repetition_penalty": 1.1}`. + - **Zero-shot clone:** add `"reference_audio_b64": ""` + `"reference_text": + ""`. Keep `repetition_penalty <= 1.1` for cloning (higher penalizes the + in-context reference audio tokens and breaks generation). +- `GET /voices`, `GET /health`, `GET /docs` (OpenAPI UI). + +**Expressive tags** (baddy is trained for these): ` + ` etc. Use **real carrier sentences with sparse, sentence-boundary tags** — +stacking many tags with little text sends this early checkpoint into a repeat-loop. + +## Deploy (irv-ml1, as `lkraven` — docker-group, no sudo) + +```bash +# one-time: stage weights (from the audition dir or a fresh pull-hf-repo) + copy the stack +mkdir -p /home/lkraven/morpheus/models +mv /home/lkraven/orpheus-audition/models/mOrpheus /home/lkraven/morpheus/models/ +mv /home/lkraven/orpheus-audition/models/snac_24khz /home/lkraven/morpheus/models/ +# copy compose.yaml + tts/ to /home/lkraven/morpheus/, cp .env.example .env +cd /home/lkraven/morpheus && docker compose build && docker compose up -d +``` + +## Gotchas + +- **Pin `vllm/vllm-openai:v0.23.0`** — `latest` ships Blackwell-only kernels (oink/aiter) + that crash on Ampere *import*. Do not bump to `latest` on this box. +- **GPU = 3090, not the A6000** — the A6000 is comfy's and spikes to ~41 GB without warning + (OOM'd two launches). FP8's ~5 GB footprint coexists with the 3090 audio zoo. +- **FP8** on Ampere is a VRAM save (upcast), no compute speedup — real-time comes from vLLM. +- Canonical copy lives here; deployed copy is `/home/lkraven/morpheus/` on irv-ml1. diff --git a/stacks/mOrpheus/compose.yaml b/stacks/mOrpheus/compose.yaml new file mode 100644 index 0000000..b8206f1 --- /dev/null +++ b/stacks/mOrpheus/compose.yaml @@ -0,0 +1,91 @@ +# mOrpheus — uncensored Orpheus TTS, permanent stack on irv-ml1. +# +# Two services: +# vllm-morpheus (GPU, 3090, FP8) — serves the mOrpheus 3B LLM; emits Orpheus audio +# tokens. Real-time: ~165 tok/s single-stream => RTF ~0.50 (2x real-time), measured. +# morpheus-tts (CPU) — SNAC-decodes those tokens to 24kHz WAV; the public /tts endpoint +# (baddy voice + optional zero-shot cloning). Kept off the GPU to avoid contention. +# +# PLACEMENT / GOTCHAS (irv-ml1, INTERNAL RESEARCH — CC-BY-NC, do NOT expose externally): +# - GPU = the 3090 (audio-zoo GPU, stabler); the A6000 is comfy's and spikes to ~41GB +# without warning (it OOM'd two launch attempts). Pin by UUID via MORPHEUS_GPU_UUID. +# - Image PINNED to vllm/vllm-openai:v0.23.0 — `latest` ships Blackwell-only kernels +# (oink/aiter) that crash on Ampere import. Do NOT bump to latest on this Ampere box. +# - bf16 REQUIRED, NOT FP8. FP8 (--quantization fp8) destroys audio-token generation on +# this model — 0 valid SNAC frames even at greedy (quant-breaks-TTS). bf16 weights are +# ~6.6GB; with --enforce-eager (no CUDA-graph capture) it fits the shared 3090 (~8GB +# free after nothing else vLLM). util 0.32 (~7.7GB of the 24GB 3090). +# All tunables in .env — edit that, not this file. + +name: morpheus + +services: + vllm-morpheus: + image: vllm/vllm-openai:v0.23.0 + container_name: vllm-morpheus + restart: unless-stopped + runtime: nvidia + ipc: host + environment: + - NVIDIA_VISIBLE_DEVICES=${MORPHEUS_GPU_UUID:-GPU-12c15b3c-0827-0bab-b0cd-31aaa37a8e72} + - CUDA_DEVICE_ORDER=PCI_BUS_ID + volumes: + - ${MORPHEUS_MODEL_DIR:-/home/lkraven/morpheus/models/mOrpheus}:/model:ro + command: + - /model + - --served-model-name + - morpheus + - --dtype + - bfloat16 + # NO --quantization: FP8 breaks audio-token generation on this model (see header). + - --enforce-eager + - --max-model-len + - "${MORPHEUS_MAX_LEN:-4096}" + - --gpu-memory-utilization + - "${MORPHEUS_GPU_UTIL:-0.32}" + - --max-num-seqs + - "${MORPHEUS_MAX_SEQS:-8}" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 300s + networks: [morpheus-net] + + morpheus-tts: + build: ./tts + image: morpheus-tts:local + container_name: morpheus-tts + restart: unless-stopped + depends_on: + vllm-morpheus: + condition: service_healthy + environment: + - VLLM_URL=http://vllm-morpheus:8000/v1/completions + - VLLM_MODEL=morpheus + - SNAC_DEVICE=cpu + - MORPHEUS_DEFAULT_VOICE=${MORPHEUS_DEFAULT_VOICE:-baddy} + - MORPHEUS_VOICES=${MORPHEUS_VOICES:-baddy} + volumes: + - ${MORPHEUS_MODEL_DIR:-/home/lkraven/morpheus/models/mOrpheus}:/model:ro + - ${SNAC_DIR:-/home/lkraven/morpheus/models/snac_24khz}:/snac:ro + ports: + - "${MORPHEUS_TTS_PORT:-8299}:8000" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + labels: + - homepage.group=AI Systems + - homepage.name=mOrpheus TTS + - homepage.icon=mdi-account-voice + - homepage.description=Uncensored Orpheus TTS (baddy + zero-shot clone), real-time, irv-ml1 + - homepage.href=http://10.100.79.3:${MORPHEUS_TTS_PORT:-8299}/docs + networks: [morpheus-net] + +networks: + morpheus-net: + name: morpheus-net diff --git a/stacks/mOrpheus/tts/Dockerfile b/stacks/mOrpheus/tts/Dockerfile new file mode 100644 index 0000000..fc62f45 --- /dev/null +++ b/stacks/mOrpheus/tts/Dockerfile @@ -0,0 +1,14 @@ +# mOrpheus TTS wrapper — CPU-only (SNAC decode + FastAPI /tts). Calls the vLLM engine. +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1 && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY requirements.txt . +# CPU torch (SNAC decode is small; keeps this container off the GPU / out of contention) +RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu \ + && pip install --no-cache-dir -r requirements.txt + +COPY app.py . +EXPOSE 8000 +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/stacks/mOrpheus/tts/app.py b/stacks/mOrpheus/tts/app.py new file mode 100644 index 0000000..8b62b2f --- /dev/null +++ b/stacks/mOrpheus/tts/app.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""mOrpheus TTS wrapper — turns the vLLM engine's Orpheus audio tokens into 24kHz WAV. + +Architecture: this CPU service builds the Orpheus prompt (as raw token ids), calls the +vLLM engine (which serves the mOrpheus LLM), recovers the generated audio-token ids by +re-tokenizing the returned text (vLLM emits them as `` strings when +skip_special_tokens=false), then SNAC-decodes them to audio. + +Token scheme (verified): audio-base 128266, 7-token SNAC frames (pos0->L1, pos1/4->L2, +pos2/3/5/6->L3, offsets k*4096). Named-speaker prompt: [SOH] "voice: text" [EOT][SOA]. +Zero-shot cloning: [BOS][SOH] ref_text [EOT][SOA][SOS] [EOS_sp] then +[SOH] text [EOT][SOA]. Generation stops at end-of-speech (128258). +""" +import os, io, base64 +import numpy as np, torch, soundfile as sf, requests +from scipy.signal import resample_poly +from fastapi import FastAPI, HTTPException +from fastapi.responses import Response +from pydantic import BaseModel +from transformers import AutoTokenizer +from snac import SNAC + +MODEL_DIR = os.environ.get("MORPHEUS_MODEL_DIR", "/model") +SNAC_DIR = os.environ.get("SNAC_DIR", "/snac") +VLLM_URL = os.environ.get("VLLM_URL", "http://vllm-morpheus:8000/v1/completions") +VLLM_MODEL = os.environ.get("VLLM_MODEL", "morpheus") +SNAC_DEVICE = os.environ.get("SNAC_DEVICE", "cpu") +DEFAULT_VOICE = os.environ.get("MORPHEUS_DEFAULT_VOICE", "baddy") +VOICES = [v for v in os.environ.get("MORPHEUS_VOICES", "baddy").split(",") if v] +AUDIO_MAX = 156937 +AUDIO_BASE, SOS, EOS_SP, SOH, SOA, EOT, BOS = 128266, 128257, 128258, 128259, 128260, 128009, 128000 + +tok = AutoTokenizer.from_pretrained(MODEL_DIR) +snac_model = SNAC.from_pretrained(SNAC_DIR).to(SNAC_DEVICE).eval() +app = FastAPI(title="mOrpheus TTS", version="0.1.0") + + +class TTSReq(BaseModel): + text: str + voice: str = DEFAULT_VOICE + temperature: float = 0.6 + top_p: float = 0.95 + max_tokens: int = 1200 + repetition_penalty: float = 1.1 # keep <=1.1 for cloning (higher penalizes ref audio tokens) + reference_audio_b64: str | None = None # optional zero-shot clone: base64 WAV + reference_text: str | None = None # transcript of the reference + + +def _encode_ref(wav_bytes: bytes, ref_text: str) -> list[int]: + wav, sr = sf.read(io.BytesIO(wav_bytes)) + if wav.ndim > 1: + wav = wav.mean(1) + wav = wav.astype(np.float32) + if sr != 24000: + wav = resample_poly(wav, 24000, sr).astype(np.float32) + wt = torch.tensor(wav, device=SNAC_DEVICE).view(1, 1, -1) + with torch.inference_mode(): + codes = snac_model.encode(wt) + L1, L2, L3 = [c.squeeze(0).tolist() for c in codes] + ids = [] + for i in range(len(L1)): + ids += [L1[i] + AUDIO_BASE, L2[2 * i] + AUDIO_BASE + 4096, L3[4 * i] + AUDIO_BASE + 8192, + L3[4 * i + 1] + AUDIO_BASE + 12288, L2[2 * i + 1] + AUDIO_BASE + 16384, + L3[4 * i + 2] + AUDIO_BASE + 20480, L3[4 * i + 3] + AUDIO_BASE + 24576] + return [BOS, SOH] + tok(ref_text, add_special_tokens=False).input_ids + [EOT, SOA, SOS] + ids + [EOS_SP] + + +def _build_prompt(req: TTSReq) -> list[int]: + if req.reference_audio_b64 and req.reference_text: + ref = _encode_ref(base64.b64decode(req.reference_audio_b64), req.reference_text) + return ref + [SOH] + tok(req.text, add_special_tokens=False).input_ids + [EOT, SOA] + return [SOH] + tok(f"{req.voice}: {req.text}").input_ids + [EOT, SOA] + + +def _decode(ids: list[int]): + if SOS in ids: + ids = ids[len(ids) - 1 - ids[::-1].index(SOS) + 1:] + codes = [t - AUDIO_BASE for t in ids if AUDIO_BASE <= t <= AUDIO_MAX] + l1, l2, l3 = [], [], [] + for i in range(len(codes) // 7): + f = codes[7 * i:7 * i + 7] + c = [f[0], f[1] - 4096, f[2] - 8192, f[3] - 12288, f[4] - 16384, f[5] - 20480, f[6] - 24576] + if any(x < 0 or x > 4095 for x in c): + continue + l1.append(c[0]); l2 += [c[1], c[4]]; l3 += [c[2], c[3], c[5], c[6]] + if not l1: + return None + ct = [torch.tensor(l1).unsqueeze(0).to(SNAC_DEVICE), + torch.tensor(l2).unsqueeze(0).to(SNAC_DEVICE), + torch.tensor(l3).unsqueeze(0).to(SNAC_DEVICE)] + with torch.inference_mode(): + return snac_model.decode(ct).squeeze().cpu().numpy() + + +@app.get("/health") +def health(): + return {"status": "ok", "voices": VOICES, "default": DEFAULT_VOICE, "engine": VLLM_URL} + + +@app.get("/voices") +def voices(): + return {"voices": VOICES, "default": DEFAULT_VOICE} + + +@app.post("/tts") +def tts(req: TTSReq): + prompt_ids = _build_prompt(req) + payload = {"model": VLLM_MODEL, "prompt": prompt_ids, "max_tokens": req.max_tokens, + "temperature": req.temperature, "top_p": req.top_p, "skip_special_tokens": False, + "stop_token_ids": [EOS_SP], "repetition_penalty": req.repetition_penalty} + try: + r = requests.post(VLLM_URL, json=payload, timeout=180) + except requests.RequestException as e: + raise HTTPException(502, f"engine unreachable: {e}") + if r.status_code != 200: + raise HTTPException(502, f"engine {r.status_code}: {r.text[:200]}") + text = r.json()["choices"][0]["text"] + gen_ids = tok(text, add_special_tokens=False).input_ids + audio = _decode(gen_ids) + if audio is None: + raise HTTPException(500, "no audio tokens generated") + buf = io.BytesIO() + sf.write(buf, audio, 24000, format="WAV", subtype="PCM_16") + return Response(content=buf.getvalue(), media_type="audio/wav", + headers={"X-Audio-Seconds": f"{len(audio)/24000:.2f}"}) diff --git a/stacks/mOrpheus/tts/requirements.txt b/stacks/mOrpheus/tts/requirements.txt new file mode 100644 index 0000000..3e72f19 --- /dev/null +++ b/stacks/mOrpheus/tts/requirements.txt @@ -0,0 +1,8 @@ +snac +transformers +soundfile +scipy +numpy +fastapi +uvicorn[standard] +requests