kokoro: persist custom voices across container recreate

Wrapper only enumerates one voice directory (settings.voices_dir,
default /app/api/src/voices/v1_0 — inside the container's writable
layer, not bind-mounted). Override via VOICES_DIR=/app/user_voices
(host bind mount) and add a command shim that cp -r's built-ins from
the in-image v1_0 into user_voices on every start. Built-ins re-seed
fresh from the image (so upgrades that add voices propagate); custom
.pt files in user_voices are preserved (cp -r is additive).

Also adds scripts/blend_kokoro_voice.py + a playbook around it that
mirrors the wrapper's request-time voice="a(w)+b(w)" math but writes
the result as a named .pt to user_voices, making it discoverable via
GET /v1/audio/voices and persistent across recreate. Defaults to
athena = af_bella(2)+af_aoede(1) normalized.

playbooks/blend-kokoro-voice.yaml

@@ -0,0 +1,78 @@
+# Blend two or more Kokoro voicepacks into a new persistent voice.
+#
+# Mirrors the wrapper's request-time `voice="a(w)+b(w)"` blend op
+# (api/src/services/tts_service.py::_get_voices_path) — same math,
+# but the result is written to /worktank/kokoro/user_voices/ on the
+# host (bind-mounted into the container at /app/user_voices, which
+# the compose's VOICES_DIR env points the wrapper at). Result:
+#   - immediately appears in GET /v1/audio/voices
+#   - usable as voice="<out_name>" in /v1/audio/speech
+#   - persistent across `up --force-recreate` and image upgrade
+#     (it's on the host bind mount, not the container layer)
+#
+# Requires the persistent-voices compose layout (entrypoint shim +
+# VOICES_DIR=/app/user_voices). If the deployed compose predates that,
+# run `scripts/elway irv-ml1 --playbook playbooks/deploy-kokoro.yaml`
+# first to roll it out.
+#
+# Usage (defaults to athena = af_bella(2)+af_aoede(1) normalized):
+#   scripts/elway irv-ml1 --playbook playbooks/blend-kokoro-voice.yaml
+#
+# Override per-run:
+#   scripts/elway irv-ml1 --playbook playbooks/blend-kokoro-voice.yaml \
+#     --var 'recipe=af_bella(1)+am_adam(1)' --var out_name=androgyne
+#
+# Re-run with --var force=1 to overwrite an existing voice.
+
+vars:
+  recipe: "af_bella(2)+af_aoede(1)"
+  out_name: athena
+  user_voices_dir: /worktank/kokoro/user_voices
+  container: kokoro
+  script_local: scripts/blend_kokoro_voice.py
+  script_host: /tmp/blend_kokoro_voice.py
+  script_in_container: /tmp/blend_kokoro_voice.py
+  force: "0"
+  api_port: "8193"
+
+steps:
+  - name: Upload blend script to host /tmp
+    upload:
+      src: "{{ script_local }}"
+      dest: "{{ script_host }}"
+      mode: "0755"
+
+  - name: Copy script into kokoro container
+    shell: docker cp {{ script_host }} {{ container }}:{{ script_in_container }}
+
+  - name: Run blend inside container
+    # `-u 0`: host /worktank/kokoro/user_voices is chowned to 1001:1001
+    # (appuser) by deploy-kokoro.yaml; we still write as root so the
+    # output is plainly 0644 root:root and the wrapper (running as
+    # appuser) can read it. Also avoids any UID-mapping subtlety.
+    # --force is wired through a shell switch so we can stay inside
+    # elway's plain {{ var }} substitution (no jinja conditionals).
+    shell: |
+      FORCE_FLAG=""
+      if [ "{{ force }}" = "1" ]; then FORCE_FLAG="--force"; fi
+      docker exec -u 0 {{ container }} python {{ script_in_container }} \
+        --recipe '{{ recipe }}' --out '{{ out_name }}' $FORCE_FLAG
+    when: "[ ! -f {{ user_voices_dir }}/{{ out_name }}.pt ] || [ '{{ force }}' = '1' ]"
+
+verify:
+  - name: Output .pt exists on host (persistent)
+    shell: test -s {{ user_voices_dir }}/{{ out_name }}.pt
+    changed_when: "false"
+
+  - name: Voice appears in /v1/audio/voices
+    shell: curl -sf http://localhost:{{ api_port }}/v1/audio/voices | grep -q '"{{ out_name }}"'
+    changed_when: "false"
+
+  - name: Synthesize a probe sample (no playback)
+    shell: |
+      curl -sf -X POST http://localhost:{{ api_port }}/v1/audio/speech \
+        -H 'Content-Type: application/json' \
+        -d '{"model":"kokoro","input":"Hello, I am {{ out_name }}.","voice":"{{ out_name }}","response_format":"wav"}' \
+        -o /tmp/{{ out_name }}-probe.wav \
+        && test -s /tmp/{{ out_name }}-probe.wav
+    changed_when: "false"

playbooks/deploy-kokoro.yaml

@@ -32,6 +32,17 @@ steps:
     shell: mkdir -p {{ user_voices_dir }}
     creates: "{{ user_voices_dir }}"
 
+  - name: Chown user-voices to 1001:1001 (appuser inside container)
+    # The compose `command:` shim runs as appuser (image USER) and
+    # cp's the in-image built-in voicepacks into this dir on every
+    # container start. appuser is uid 1001 inside the container; the
+    # host has no user with that uid, so we chown numerically. Once
+    # set, the shim can write; the blend script (run via docker exec
+    # -u 0) writes as root, producing 0644 files appuser can read.
+    shell: chown 1001:1001 {{ user_voices_dir }}
+    sudo: true
+    when: '[ "$(stat -c %u:%g {{ user_voices_dir }})" != "1001:1001" ]'
+
   - name: Ensure voices dir exists (used only if compose mount is enabled)
     shell: mkdir -p {{ voices_dir }}
     creates: "{{ voices_dir }}"

scripts/blend_kokoro_voice.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# Blend Kokoro voicepacks into a new persistent voice.
+#
+# Mirrors Kokoro-FastAPI's request-time blend op (see
+# api/src/services/tts_service.py::_get_voices_path) so the resulting
+# .pt behaves identically to the equivalent inline `voice="a(w)+b(w)"`
+# request — except it lives on disk, gets a stable name, and is
+# discoverable via GET /v1/audio/voices.
+#
+# Designed to be run INSIDE the kokoro container (where torch and the
+# built-in voicepacks are already present):
+#
+#   docker exec -u 0 kokoro python /tmp/blend_kokoro_voice.py \
+#     --recipe 'af_bella(2)+af_aoede(1)' --out athena
+#
+# I/O layout (paired with the compose changes that ship custom-voice
+# persistence):
+#
+#   * Reads from /app/user_voices/, which the compose `command:` shim
+#     pre-seeds with the in-image built-ins on every container start.
+#     So both built-ins (af_bella, af_aoede, ...) and any prior custom
+#     blends resolve from the same directory.
+#   * Writes to /app/user_voices/<out>.pt — that path is the host
+#     bind mount /worktank/kokoro/user_voices/, so the result is
+#     immediately discoverable (the wrapper has VOICES_DIR pointed
+#     here) AND fully persistent across container recreate, image
+#     upgrade, and host reboot.
+import argparse
+import os
+import re
+import sys
+
+import torch
+
+VOICES_DIR = "/app/user_voices"
+
+
+def resolve(name: str) -> str:
+    p = os.path.join(VOICES_DIR, f"{name}.pt")
+    if os.path.exists(p):
+        return p
+    raise FileNotFoundError(f"voice {name!r} not found in {VOICES_DIR}")
+
+
+def parse_recipe(recipe: str):
+    parts = re.split(r"([-+])", recipe.replace(" ", ""))
+    terms = []
+    for i, tok in enumerate(parts):
+        if i % 2 == 0:
+            if "(" in tok and ")" in tok:
+                name = tok.split("(")[0]
+                weight = float(tok.split("(")[1].split(")")[0])
+            else:
+                name, weight = tok, 1.0
+            sign = +1.0 if i == 0 or parts[i - 1] == "+" else -1.0
+            terms.append((name, sign * weight))
+    return terms
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--recipe", required=True,
+                    help='e.g. "af_bella(2)+af_aoede(1)"')
+    ap.add_argument("--out", required=True, help="output voice name (no .pt)")
+    ap.add_argument("--no-normalize", action="store_true",
+                    help="sum weights as-is instead of dividing by total")
+    ap.add_argument("--force", action="store_true",
+                    help="overwrite existing output voice")
+    args = ap.parse_args()
+
+    terms = parse_recipe(args.recipe)
+    if not terms:
+        sys.exit("empty recipe")
+
+    total = sum(abs(w) for _, w in terms) if not args.no_normalize else 1.0
+    if total == 0:
+        sys.exit("weights sum to zero")
+
+    out_path = os.path.join(VOICES_DIR, f"{args.out}.pt")
+    if os.path.exists(out_path) and not args.force:
+        sys.exit(f"refusing to overwrite {out_path} (pass --force)")
+
+    blended = None
+    for name, w in terms:
+        t = torch.load(resolve(name), map_location="cpu") * (w / total)
+        blended = t if blended is None else blended + t
+
+    torch.save(blended, out_path)
+    print(f"wrote {out_path} (shape={tuple(blended.shape)}, dtype={blended.dtype})")
+
+
+if __name__ == "__main__":
+    main()

stacks/kokoro/README.md

@@ -59,10 +59,36 @@ Supported `response_format`: `mp3 | wav | opus | flac | pcm`.
   Discoverable via `GET /v1/audio/voices` — naming convention is
   `<lang_code><gender_letter>_<name>` (e.g. `af_bella`, `am_adam`,
   `jf_alpha`, `zf_xiaobei`).
-- **Custom**: drop `.pt` voicepacks into `/worktank/kokoro/user_voices/`
-  on the host. Wrapper auto-discovers them on next request (no
-  restart). Training Kokoro voices is non-trivial — consult the
-  hexgrad community for how-to.
+- **Custom blends**: weighted-average existing voicepacks into a new
+  named voice using `playbooks/blend-kokoro-voice.yaml` (script:
+  `scripts/blend_kokoro_voice.py`). Mirrors the wrapper's request-time
+  `voice="a(w)+b(w)"` math but persists the result so it shows up in
+  `/v1/audio/voices`. Example shipped: `athena = af_bella(2)+af_aoede(1)`
+  normalized.
+
+  ```bash
+  scripts/elway irv-ml1 --playbook playbooks/blend-kokoro-voice.yaml \
+    --var 'recipe=af_bella(1)+am_adam(1)' --var out_name=androgyne
+  # add --var force=1 to overwrite an existing voice
+  ```
+
+  Persistence design: the wrapper enumerates exactly one directory
+  (`VOICES_DIR`, default in-image `/app/api/src/voices/v1_0`).
+  compose.yaml overrides `VOICES_DIR=/app/user_voices` (host bind
+  mount `/worktank/kokoro/user_voices`) and adds a `command:` shim
+  that `cp -r`'s the in-image built-ins into that dir on every
+  container start. Result:
+  * built-in voicepacks re-seed on each start, so image upgrades that
+    add/change built-ins propagate automatically
+  * custom blends written by the script live on the host bind mount —
+    survive `docker restart`, `up --force-recreate`, image upgrade,
+    and host reboot
+  * one-time prereq: the host dir must be chowned to `1001:1001`
+    (uid of `appuser` inside the container) so the shim's cp can
+    write — handled by `playbooks/deploy-kokoro.yaml`
+- **Cloned voices**: training a Kokoro voice from samples is non-
+  trivial — consult the hexgrad community for how-to. For voice
+  cloning use Chatterbox Turbo or IndexTTS-2 instead.
 
 ## Deploy
 

stacks/kokoro/compose.yaml

@@ -27,18 +27,30 @@ services:
     # that env var harmlessly; the gpu variant honors it.
     ports:
       - "${KOKORO_BIND:-0.0.0.0}:${KOKORO_PORT}:8880"
+    # Persistent custom voices: the wrapper only enumerates one voice
+    # directory (controlled by VOICES_DIR; default in-image is
+    # /app/api/src/voices/v1_0 which is NOT bind-mounted). We point it
+    # at /app/user_voices (which IS host-bind-mounted) and seed the
+    # built-ins into it on every container start. Custom voices written
+    # by scripts/blend_kokoro_voice.py land directly on the host bind
+    # mount and survive `up --force-recreate` and image upgrade.
+    # `cp -r` (no `-n`): always refresh built-ins from the image so
+    # upgrades that add/change built-in voicepacks propagate.
+    # Custom-named .pt files in user_voices that don't exist in v1_0
+    # are NOT touched (cp -r only copies entries from src).
+    command: ["/bin/bash", "-c", "cp -r /app/api/src/voices/v1_0/. /app/user_voices/ && exec ./entrypoint.sh"]
     environment:
       - NVIDIA_VISIBLE_DEVICES=${KOKORO_GPU_DEVICES:-}
       - USE_GPU=${KOKORO_USE_GPU:-false}
       - API_LOG_LEVEL=${KOKORO_LOG_LEVEL:-INFO}
+      - VOICES_DIR=/app/user_voices
     volumes:
-      # Optional voice-overlay mount — drop a custom <name>.pt into the
-      # host dir to make it available alongside the 60+ built-ins. The
-      # image already ships voicepacks at this path, so the bind mount
-      # SHADOWS them — only do this if you actually want to manage the
-      # full voice library yourself. For most deploys, leave the mount
-      # commented out and use the in-image voices.
-      # - ${KOKORO_VOICES_DIR}:/app/api/src/voices/v1_0
+      # Host-bind-mounted voice directory. Hosts both the image's
+      # built-ins (re-seeded on each container start by the command
+      # shim above) and any custom voices created via the blend
+      # script. Must be chowned to uid 1001 (appuser inside the
+      # container) on the host so the shim's cp can write — handled
+      # by playbooks/deploy-kokoro.yaml.
       - ${KOKORO_USER_VOICES_DIR}:/app/user_voices
     healthcheck:
       # The image is python-based with curl available. /v1/audio/voices