esh-pfi-infrastructure/stacks/gen-seat/compose.yaml

# gen-seat — the fleet's general `gen` seat on fv-ml1 GPU 0 (:8015).
# Serves JonathanColetti/Qwen3.8-27B-Uncensored (Heretic abliteration KL 0.12, vision-intact
# Qwen3_5ForConditionalGeneration, grafted MTP head), quantized in-house to NVFP4 W4A16
# (compressed-tensors) with the bf16 MTP grafted back. ⚠ the grafted MTP requires
# `re:^mtp.*` in config.json quantization_config.ignore or vLLM loads it uninitialized (0% accept).
# Backs gateway aliases: gen, gen-reasoning, summarizer, summarizer-large, classifier,
# image-judge, qwen-image-bench (all via api_base :8015).
#
# Two served-names (base + `-thinking`): LiteLLM keys deployments by (model, api_base), so gen
# and gen-reasoning use distinct names to avoid the shared-config enable_thinking clobber.
# --mamba-cache-dtype float32 for the GDN/hybrid-linear-attn state; MTP via qwen3_5_mtp n=3;
# full 262K context (hybrid attention -> cheap KV, ~8.6GB at 262K). All tunables in .env.
# (Renamed from qwen36-27b-aeon/vllm-aeon-gen; dormant RP service dropped 2026-08-15.)

name: gen-seat

services:
  vllm-gen:
    image: ${GEN_IMAGE:-vllm/vllm-openai:latest}
    container_name: ${GEN_CONTAINER_NAME:-vllm-gen}
    restart: unless-stopped
    ipc: host
    ports:
      - "${GEN_PORT:-8015}:8000"
    volumes:
      - /tank/aimodels/huggingface:/hfcache
      - ${GEN_MODEL:-/tank/aimodels/qwen38-27b-uncensored-nvfp4}:/model:ro
    environment:
      - HF_HOME=/hfcache
      - HF_HUB_CACHE=/hfcache/hub
      - VLLM_API_KEY=${API_KEY:-}
      - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
    command:
      - /model
      - --served-model-name
      - ${GEN_SERVED_NAME:-qwen3.8-27b-uncensored}
      - ${GEN_SERVED_NAME_THINK:-qwen3.8-27b-uncensored-thinking}
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      - --quantization
      - ${GEN_QUANT:-compressed-tensors}
      - --gpu-memory-utilization
      - ${GEN_GPU_MEM_UTIL:-0.45}
      - --max-model-len
      - ${GEN_MAX_MODEL_LEN:-262144}
      - --max-num-seqs
      - ${GEN_MAX_NUM_SEQS:-16}
      - --max-num-batched-tokens
      - "16384"
      - --trust-remote-code
      - --dtype
      - auto
      - --mamba-cache-dtype
      - float32
      - --kv-cache-dtype
      - ${GEN_KV_CACHE_DTYPE:-fp8}
      # Prefix caching ON (vLLM V1 default). The APC-off MTP mitigation was tried
      # 2026-08-17 and did NOT hold — synthetic 7-turn probes passed but the
      # operator still saw severe degeneration in real use, so it was reverted.
      # See the MTP block below.
      - --enable-prefix-caching
      - --enable-chunked-prefill
      - --limit-mm-per-prompt
      - '{"image": 4}'
      - --reasoning-parser
      - ${GEN_REASONING_PARSER:-qwen3}
      # Seat-level DEFAULT reasoning effort. Per-request overridable via
      # chat_template_kwargs; an invalid value 400s rather than silently
      # falling back (the template validates and raises).
      #
      # WHY: the Qwen3.8 chat template defaults to `xhigh`
      # (chat_template.jinja:47), and at xhigh reasoning GROWS WITH
      # CONVERSATION DEPTH and has a long tail. Measured 2026-08-16 on THIS
      # model (not inferred from a card), 3-turn history, 3 samples each:
      #
      #     xhigh (default)  4,529-5,532 chars, observed spiking to 9,261
      #     medium           2,602-3,283 chars
      #     low              1,657-2,505 chars
      #
      # Content length was unchanged or BETTER at medium (median 335w vs 343w),
      # so this is ~40% less budget burned for the same answer. Single-turn
      # probes do not surface it -- single-turn reasoning is only ~1-1.8k chars,
      # which is why the acceptance gate missed it entirely.
      #
      # The failure it prevents: at xhigh the reasoning tail exhausts max_tokens
      # before any content is emitted, returning finish_reason=length with an
      # EMPTY content field. That reads as the model being degenerate on long
      # conversations when it is a budget interaction. Pairs with the
      # gateway-chat max_tokens default (raised 1024 -> 4096); NEITHER fix alone
      # is sufficient.
      #
      # Only set this for a template that actually reads the kwarg -- the
      # Qwen3.6 line (Fable-Fusion on char-rp-reasoning) ignores it entirely,
      # where it would be a false lever.
      - --default-chat-template-kwargs
      - '{"reasoning_effort": "${GEN_REASONING_EFFORT:-medium}"}'
      - --enable-auto-tool-choice
      - --tool-call-parser
      - qwen3_coder
      # ⚠️ MTP / SPECULATIVE DECODING DISABLED — the verified known-good.
      # Re-enable ONLY after real-use multi-turn testing, not a synthetic probe.
      #
      # History: MTP-off is clean. The APC-off mitigation (keep MTP, disable
      # prefix caching) passed synthetic 7-turn probes on 2026-08-17 but the
      # operator STILL saw severe degeneration in real use — so a passing probe
      # is NOT sufficient evidence; reverted to MTP-off. The synthetic probes
      # structurally under-cover real workloads (content distribution, depth).
      #
      # ROOT CAUSE (isolated 2026-08-16, operator-confirmed): `qwen3_5_mtp`
      # speculative decoding CORRUPTS Qwen3.8-27B output in multi-turn
      # conversations once cumulative context passes ~2,000 tokens — CONTEXT-
      # BLEEDING (a "describe durian" reply that contained the Krebs-cycle and
      # winter answers from earlier turns), then collapse. Confirmed by two
      # cross-frontier research peers as a KNOWN upstream bug: the Gated-DeltaNet
      # recurrent state cannot roll back on a partial draft-accept, so the SSM
      # state corrupts (vllm#47087 symptom-twin, #43559 fix lineage). It is
      # architectural — vLLM, SGLang, and llama.cpp mainline all share it — and
      # NOT depth-dependent (n=1 and n=2 both corrupt, so num_speculative_tokens=1
      # is NOT a fix).
      #
      # The APC-off mitigation (#43559/#47194) was supposed to make the GDN
      # align-path bug inert while keeping MTP. It did not survive real use.
      # The proper upstream fix (#51113) is in vLLM main / v0.27.2rc0 only — not
      # in a stable release — so we do not jump the fleet gateway to an RC.
      # Cost of MTP-off: ~half decode tok/s. Accepted as the known-good.
      #
      # MTP ON — PRIMARY GEN CONFIG (resolved 2026-08-17, operator-confirmed
      # coherent). The whole multi-day degeneration hunt root-caused to the AEON
      # W4A4 quant being DEFECTIVE: it went degenerate ~15-20% of generations and
      # forced a regenerate. MTP / prefix-caching / gateway all merely AMPLIFIED
      # it, which is why MTP-off and the vLLM #51113 fix each "helped" without
      # fixing it. This build — the in-house JonathanColetti/Heretic mixed
      # NVFP4+FP8 weights (FP8 attention, NOT W4A4) on vLLM nightly (#51113 fix)
      # with MTP ON — is coherent through long multi-turn in real use. AEON
      # (qwen38-27b-aeon-ultimate-nvfp4) was PURGED as no-good. Primary gen until
      # the DavidAU Qwen3.8 lands.
      - --speculative-config
      - '{"method": "${GEN_SPEC_METHOD:-qwen3_5_mtp}", "num_speculative_tokens": ${GEN_SPEC_TOKENS:-3}}'
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids:
                - "${GEN_GPU_ID:-0}"
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 900s
    networks:
      - tnet
    labels:
      - homepage.group=AI - Inference
      - homepage.name=Qwen3.8-27B Uncensored (NVFP4, vision, MTP) — gen
      - homepage.icon=mdi-creation
      - homepage.description=Uncensored Qwen3.8-27B multimodal NVFP4+MTP, the `gen` seat (fv-ml1 GPU 0)
      - homepage.href=http://10.251.50.54:${GEN_PORT:-8015}/docs

networks:
  tnet:
    name: traefik-net
    external: true