esh-pfi-infrastructure/playbooks/deploy-fish-s2.yaml

# Deploy Fish Audio S2-Pro (richest paralinguistic open-source TTS) to
# irv-ml1.
#
# Builds the image locally from fishaudio/fish-speech via docker buildx
# git URL context. ~10-15 min cold build (CUDA 12.x + torch + flash-attn
# + Fish's training/inference deps). First start downloads s2-pro
# (~9 GB BF16) into the bind-mounted HF cache. Generous /v1/health
# wait deadline accommodates both.
#
# Usage:
#   scripts/elway irv-ml1 --playbook playbooks/deploy-fish-s2.yaml
#
# Idempotent — every step is creates-/when-gated; rerun is safe.

vars:
  compose_dir: /opt/docker/compose/fish-s2
  references_dir: /worktank/fish-s2/references
  checkpoints_dir: /worktank/fish-s2/checkpoints
  cache_dir: /worktank/fish-s2/hf_cache
  host_port: "8195"

steps:
  # ── host-side dirs ──────────────────────────────────────────────────

  - name: Ensure /worktank/fish-s2 root exists (one-time, sudo)
    shell: mkdir -p /worktank/fish-s2
    sudo: true
    creates: /worktank/fish-s2

  - name: Chown /worktank/fish-s2 to lkraven
    shell: chown -R lkraven:lkraven /worktank/fish-s2
    sudo: true
    when: "[ \"$(stat -c %U /worktank/fish-s2)\" != \"lkraven\" ]"

  - name: Ensure references dir exists
    shell: mkdir -p {{ references_dir }}
    creates: "{{ references_dir }}"

  - name: Ensure checkpoints dir exists
    shell: mkdir -p {{ checkpoints_dir }}
    creates: "{{ checkpoints_dir }}"

  - name: Ensure HF cache dir exists
    shell: mkdir -p {{ cache_dir }}
    creates: "{{ cache_dir }}"

  - name: Ensure compose dir exists
    shell: mkdir -p {{ compose_dir }}
    creates: "{{ compose_dir }}"

  # ── deploy compose + env ────────────────────────────────────────────

  - name: Upload compose.yaml
    upload:
      src: stacks/fish-s2/compose.yaml
      dest: "{{ compose_dir }}/compose.yaml"
      mode: "0644"

  - name: Seed .env from template (only if absent)
    upload:
      src: stacks/fish-s2/.env.example
      dest: "{{ compose_dir }}/.env"
      mode: "0644"
    when: "[ ! -f {{ compose_dir }}/.env ]"

  # ── pre-pull model checkpoint ───────────────────────────────────────
  # Fish doesn't auto-download on first run — start_server.sh validates
  # checkpoints/s2-pro/ exists and exits cleanly (rc=0) if missing. So
  # we pre-pull fishaudio/s2-pro into the bind-mount via a one-shot
  # python container with hf_transfer (~9 GB at ~100 MB/s).
  # Idempotent — `creates:` skips if the codec file is already there.

  - name: Pre-pull fishaudio/s2-pro into checkpoints (~9 GB, ~90s)
    shell: |
      docker run --rm --user 1000:1000 \
        -e HOME=/tmp/h -e HF_HUB_ENABLE_HF_TRANSFER=1 \
        -v {{ checkpoints_dir }}:/dest \
        python:3.12-slim sh -c 'set -e; mkdir -p /tmp/h /tmp/pip /tmp/site; PIP_CACHE_DIR=/tmp/pip pip install --quiet --target /tmp/site huggingface_hub hf_transfer; PYTHONPATH=/tmp/site python -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id=\"fishaudio/s2-pro\", local_dir=\"/dest/s2-pro\", local_dir_use_symlinks=False, ignore_patterns=[\"*.md\",\"*.png\",\".gitattributes\"])"'
    creates: "{{ checkpoints_dir }}/s2-pro/codec.pth"

  # ── build + bring up ────────────────────────────────────────────────

  - name: docker compose build (~10-15 min first time; cached after)
    shell: |
      set -o pipefail
      cd {{ compose_dir }} && docker compose build 2>&1 \
        | grep -vE '^#[0-9]+ |^ => |^=> |Collecting|Downloading|Requirement|Using cached|Installing collected|Successfully (installed|built)|━'

  - name: docker compose up -d
    shell: cd {{ compose_dir }} && docker compose up -d

  - name: Wait for /v1/health to respond (allow ~15 min for first model download + warmup)
    shell: |
      for i in $(seq 1 180); do
        curl -sf -o /dev/null --max-time 3 http://localhost:{{ host_port }}/v1/health && exit 0
        sleep 5
      done
      exit 1
    changed_when: "false"

verify:
  - name: /v1/health returns 200
    shell: curl -sf -o /dev/null http://localhost:{{ host_port }}/v1/health
    changed_when: "false"

  - name: /v1/tts returns a real WAV (POST with text body)
    # Fish's API is NOT OpenAI-compatible — there's no /v1/audio/speech
    # and no /v1/audio/voices. The single TTS endpoint is POST /v1/tts
    # with at minimum {"text":"..."} returning audio/wav. Voice cloning
    # is via reference= field in the body (paths under /app/references).
    # Verify by POST + asserting the response is a real RIFF WAV.
    shell: |
      out=$(mktemp --suffix=.wav)
      curl -sf -X POST http://localhost:{{ host_port }}/v1/tts \
        -H 'Content-Type: application/json' \
        -d '{"text":"Verify."}' \
        -o "$out" --max-time 30
      file -b "$out" | grep -q '^RIFF.*WAVE'
      rm -f "$out"
    changed_when: "false"

  - name: Container is running
    shell: docker inspect fish-s2 --format '{{.State.Status}}' | grep -q running
    changed_when: "false"