esh-pfi-infrastructure/stacks/gen-small-seat/compose.yaml

# gen-small-seat — a fast, cheap "gen-small" seat: Qwen3.6-35B-A3B (3B active MoE),
# Heretic-abliterated, on fv-ml1 GPU 1, :8026.
#
# WHY: a tiny-active (A3B = ~3B active of 35B total) gen tier for high-volume / low-latency
# work, complementing the big gen seat (flash-next :8022). Operator-directed 2026-09-14.
# "3.8 if possible, 3.6 if not" — there is NO general Qwen3.8 A3B (the 3.8 MoEs are
# Flash-Next and the 2.4T), so this is the 3.6-35B-A3B fallback, which is the current-gen A3B.
#
# MODEL: /tank/aimodels/qwen36-35b-a3b-heretic-nvfp4 =
#   llmfan46/Qwen3.6-35B-A3B-uncensored-heretic-Native-MTP-Preserved-NVFP4-Experts-Only
#   - Heretic v1.3.0 (MPOA — Magnitude-Preserving Orthogonal Ablation), the operator's
#     house-favorite abliteration method: 88% fewer refusals (10/100 vs 83/100) at
#     0.0015 KL divergence (quality preserved).
#   - modelopt NVFP4, experts-only (256 experts NVFP4; attn/shared-expert/linear_attn bf16).
#   - 19 MTP tensors preserved (native), so speculative decoding is available.
#   - hybrid arch (Qwen3_5MoeForConditionalGeneration): linear_attn (GDN) + attention +
#     MoE, native context 262144.
#
# ⚠ NVFP4 MoE serving: this is modelopt NVFP4 on the experts. If the NVFP4 MoE kernel
#   refuses ("does not support the deployment configuration ... kernel does not support
#   current device"), the erp-seat lesson applies (Marlin fallback) — but this is Blackwell
#   sm120 with native FP4, so it should take the native path. Read the boot log.
#
# ⚠ MTP on an A3B MoE is UNVERIFIED on this build — abliteration can desync an MTP head, and
#   MoE MTP has its own failure modes. GEN_SMALL_SPEC_CONFIG defaults to qwen3_5_mtp k=3;
#   gate it on a MEASURED acceptance (>=~40%) via a coherence + acceptance smoke, exactly as
#   cyberprev/flash-next were. To turn MTP off, delete the two --speculative-config lines.
name: gen-small-seat

services:
  vllm-gen-small:
    image: ${GEN_SMALL_IMAGE:-vllm/vllm-openai:nightly-e9d1398d9edfd90fcc1cf783805240e3effec013}
    container_name: ${GEN_SMALL_CONTAINER_NAME:-vllm-gen-small}
    restart: unless-stopped
    ipc: host
    ports:
      - "${GEN_SMALL_PORT:-8026}:8000"
    volumes:
      - /tank/aimodels/huggingface:/hfcache
      - ${GEN_SMALL_MODEL:-/tank/aimodels/qwen36-35b-a3b-heretic-nvfp4}:/model:ro
    environment:
      - VLLM_API_KEY=${API_KEY:-}
      - PYTORCH_CUDA_ALLOC_CONF=${GEN_SMALL_ALLOC_CONF:-}
    command:
      - /model
      - --served-model-name
      - ${GEN_SMALL_SERVED_NAME:-gen-small}
      - ${GEN_SMALL_SERVED_NAME_THINK:-gen-small-thinking}
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      - --quantization
      - ${GEN_SMALL_QUANT:-modelopt_fp4}
      - --gpu-memory-utilization
      - "${GEN_SMALL_GPU_MEM_UTIL:-0.55}"
      # KV pinned in bytes — GPU 0 is shared with cyberprev, so an explicit KV budget
      # avoids the util-fraction fight between co-resident seats. fp8 KV + an A3B's small
      # attention footprint make this cheap; tuned to a sane concurrency band, not max.
      - --kv-cache-memory
      - "${GEN_SMALL_KV_CACHE_MEMORY:-5905580032}"
      # Default output cap (2026-10-05, Prime). A request that OMITS max_tokens stops at 32,768
      # instead of max_model_len - prompt; an explicit max_tokens still wins (vLLM get_max_tokens:
      # request value first, generation-config max_new_tokens only as the fallback). Reason: 9
      # runaway replies in 7 days, each ~260k tokens over 20+ min (albok-nemi, all-agents-local),
      # while the longest normal reply was 6,357 tokens.
      - --override-generation-config
      - '{"max_new_tokens": 32768}'
      - --max-model-len
      - "${GEN_SMALL_MAX_MODEL_LEN:-262144}"
      - --max-num-seqs
      - "${GEN_SMALL_MAX_NUM_SEQS:-16}"
      - --max-num-batched-tokens
      - "${GEN_SMALL_MAX_NUM_BATCHED_TOKENS:-4096}"
      - --trust-remote-code
      - --dtype
      - auto
      # Hybrid backbone carries linear_attn (GDN); its state wants float32.
      - --mamba-cache-dtype
      - float32
      - --kv-cache-dtype
      - ${GEN_SMALL_KV_CACHE_DTYPE:-fp8}
      - --enable-prefix-caching
      - --enable-chunked-prefill
      - --reasoning-parser
      - ${GEN_SMALL_REASONING_PARSER:-qwen3}
      - --default-chat-template-kwargs
      - '{"reasoning_effort": "${GEN_SMALL_REASONING_EFFORT:-medium}"}'
      - --enable-auto-tool-choice
      - --tool-call-parser
      - ${GEN_SMALL_TOOL_PARSER:-qwen3_xml}
      # MTP k=3 — verify acceptance on THIS build before trusting it (see header).
      - --speculative-config
      - '${GEN_SMALL_SPEC_CONFIG:-{"method": "qwen3_5_mtp", "num_speculative_tokens": 3}}'
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids:
                - "${GEN_SMALL_GPU_ID:-0}"
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 900s
    networks:
      - tnet
    labels:
      - homepage.group=AI - Inference
      - homepage.name=gen-small (Qwen3.6-35B-A3B heretic)
      - homepage.icon=mdi-lightning-bolt
      - homepage.description=Fast A3B (3B-active) Heretic-abliterated gen seat, MTP, fv-ml1 GPU 0
      - homepage.href=http://10.251.50.54:${GEN_SMALL_PORT:-8026}/docs

networks:
  tnet:
    name: traefik-net
    external: true