esh-pfi-infrastructure/stacks/mog-sec/compose.yaml

# mog-sec — the pen-test seat on ana-ml2 GPU 1 (:8019), in the slot the retired
# fablefusion-charrp-probe used to occupy.
#
# Serves Blackfrost-Research/M.O.G.-SEC-27B-1M-CTX-BF16 (stock-Qwen3.8-27B-based,
# vision-intact Qwen3_5ForConditionalGeneration, base-graft MTP head), quantized
# in-house to mixed NVFP4 W4A4 (bulk MLP) + FP8 W8A8 (attn / linear_attn / lm_head /
# top MLP layers), with the bf16 MTP head grafted back. ⚠ the grafted MTP requires
# `re:^mtp.*` in config.json quantization_config.ignore or vLLM loads it uninitialised
# (0% accept) — handled by post_quant.py, verified present at build time.
#
# CONTEXT: served at native 262K, NOT the card's 1M. The 1M needs YaRN rope_scaling
# (absent from the weights' config) plus the SGLang/DFlash2 attention path the repo
# ships a deployment kit for — neither is our vLLM serving surface. 262K is the honest
# native ceiling here; a real 1M seat would be a separate SGLang project.
#
# Two served-names (base + `-thinking`): LiteLLM keys deployments by (model, api_base),
# so mog-sec and mog-sec-reasoning use distinct names to avoid the shared-config
# enable_thinking clobber. Same pinned nightly as the gen seat (carries the #51113
# qwen3_5_mtp x GDN partial-accept fix that the MTP-on config depends on).

name: mog-sec

services:
  vllm-mog-sec:
    image: ${MOG_IMAGE:-vllm/vllm-openai:latest}
    container_name: ${MOG_CONTAINER_NAME:-vllm-mog-sec}
    restart: unless-stopped
    ipc: host
    ports:
      - "${MOG_PORT:-8019}:8000"
    volumes:
      - /tank/aimodels/huggingface:/hfcache
      - ${MOG_MODEL:-/tank/aimodels/mog-sec-27b-nvfp4-mixed}:/model:ro
      # DFlash2 speculative drafter. Mounted unconditionally — it is inert if
      # MOG_SPEC_CONFIG selects an MTP method that does not reference /drafter.
      - ${MOG_DRAFT_MODEL:-/tank/aimodels/qwen38-27b-dflash2-drafter}:/drafter:ro
    environment:
      - HF_HOME=/hfcache
      - HF_HUB_CACHE=/hfcache/hub
      - VLLM_API_KEY=${API_KEY:-}
      # ⚠ DEFAULTS TO UNSET, deliberately. This seat previously hardcoded
      # PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True. The DFlash2 config
      # validated 2026-08-22 ran WITHOUT it, and the quant playbook §3.10
      # records expandable_segments corrupting retained tensors in another
      # context. Do not re-enable it casually — that would ship a variable the
      # tested configuration did not have.
      - PYTORCH_CUDA_ALLOC_CONF=${MOG_ALLOC_CONF:-}
    command:
      - /model
      - --served-model-name
      - ${MOG_SERVED_NAME:-mog-sec-27b}
      - ${MOG_SERVED_NAME_THINK:-mog-sec-27b-thinking}
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      - --quantization
      - ${MOG_QUANT:-compressed-tensors}
      - --gpu-memory-utilization
      - ${MOG_GPU_MEM_UTIL:-0.44}
      - --max-model-len
      - ${MOG_MAX_MODEL_LEN:-262144}
      - --max-num-seqs
      - ${MOG_MAX_NUM_SEQS:-16}
      - --max-num-batched-tokens
      # ⚠ Raising this costs peak-activation VRAM straight out of the KV pool
      # (measured 2026-08-22: 16384 -> 32768 cost ~3 GiB of KV for no benefit).
      - ${MOG_MAX_NUM_BATCHED_TOKENS:-16384}
      - --trust-remote-code
      - --dtype
      - auto
      - --mamba-cache-dtype
      - float32
      - --kv-cache-dtype
      - ${MOG_KV_CACHE_DTYPE:-fp8}
      - --enable-prefix-caching
      - --enable-chunked-prefill
      - --limit-mm-per-prompt
      - '${MOG_LIMIT_MM:-{"image": 4}}'
      # ⚠ MANDATORY on a newer vLLM. The model's own preprocessor_config.json
      # declares size.longest_edge = 16777216 px (4096x4096), which expands to
      # 16384 image tokens — one image eating 6% of a 262K context, and enough
      # to kill startup on builds that enforce the text-vs-ids count check.
      # This caps the dummy profiling image AND real images. Cost scales as
      # (edge/patch)^2 / merge^2, so 2048x2048 -> ~5125 tokens.
      - --mm-processor-kwargs
      - '${MOG_MM_PROCESSOR_KWARGS:-{"size": {"longest_edge": 4194304, "shortest_edge": 65536}}}'
      - --reasoning-parser
      - ${MOG_REASONING_PARSER:-qwen3}
      - --default-chat-template-kwargs
      - '{"reasoning_effort": "${MOG_REASONING_EFFORT:-medium}"}'
      - --enable-auto-tool-choice
      - --tool-call-parser
      - qwen3_coder
      # ONE env var carrying the whole JSON, because the two speculative shapes
      # are not interchangeable: dflash needs a "model" pointing at the drafter,
      # MTP must NOT have one. A method+tokens template cannot express both.
      #   DFlash2 : {"method": "dflash", "model": "/drafter", "num_speculative_tokens": 7}
      #   MTP     : {"method": "qwen3_5_mtp", "num_speculative_tokens": 3}
      # ⚠ Comparing the two requires matching num_speculative_tokens — see the
      # quant playbook §5.1: MTP runs a single-module head autoregressively, so
      # deeper k improves acceptance and DESTROYS throughput.
      - --speculative-config
      - '${MOG_SPEC_CONFIG:-{"method": "qwen3_5_mtp", "num_speculative_tokens": 3}}'
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids:
                - "${MOG_GPU_ID:-1}"
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 900s
    networks:
      - tnet
    labels:
      - homepage.group=AI - Inference
      - homepage.name=M.O.G.-SEC 27B (pen-test)
      - homepage.icon=mdi-shield-lock
      - homepage.description=Uncensored security model, Qwen3.8-27B NVFP4+MTP, 262K — the `mog-sec` seat (ana-ml2 GPU 1)
      - homepage.href=http://10.250.50.54:${MOG_PORT:-8019}/docs

networks:
  tnet:
    name: traefik-net
    external: true