esh-pfi-infrastructure/stacks/cyberprev-seat/compose.yaml

# cyberprev-seat — hotdogs/Qwen3.8-27B-abliterated-cyber-preview, fv-ml1 GPU 0, :8025.
#
# The THIRD sec-seat candidate, and the one that displaced sentinel-r3 (operator instruction
# 2026-09-14). Lineage of the three, because the distinction is the whole point of the A/B:
#   mog-sec     — persona system prompt on STOCK Qwen3.8-27B weights
#   sentinel-r3 — a REAL SFT pentest finetune of STOCK weights (proprietary licence); RETIRED
#   cyberprev   — a cyber offensive-security agentic TOOL-CALLING LoRA finetune
#                 (base_model_relation: finetune, merged @ scale 1.0), trained on
#                 hotdogs/cyber-sft-agent-qwen38 (8,400 rows, 22 pentest tools) on top of
#                 an ABLITERATED base (hotdogs/Qwen3.8-27B-abliterated). So: abliteration is
#                 INHERITED from the base; the NEW capability here is the cyber SFT itself.
#                 All three are Apache-2.0 except sentinel. cyberprev "will not refuse" per
#                 its card and emits <tool_call> XML for nmap/sqlmap/metasploit/hydra/etc.
#
# Quantized in-house to the same mixed recipe as mog-sec / sentinel / gen:
# NVFP4 W4A4 on MLP layers 0-55 + FP8 W8A8 on attn / linear_attn / lm_head / MLP 56-63,
# FP8 KV, vision tower + linear_attn norms + `re:^mtp.*` ignored.
# Pipeline: services/gen-seat-mixed-quant/. Source bf16: /tank/aimodels/cyberprev-bf16.
#
# ⚠ `re:^mtp.*` MUST be present in config.json quantization_config.ignore, or vLLM loads the
#   grafted BF16 MTP head UNINITIALISED and speculative decoding silently runs at 0%
#   acceptance. llm-compressor PRUNES ignore entries that matched no module at quant time,
#   and the wrapper class never loads the MTP head — so the entry is dropped unless
#   post_quant.py re-injects it. post_quant.py verifies rather than assumes; do not skip it.
#
# ⚠ max-model-len 163840, NOT the native 262K — inherited from mog-sec the hard way. Same
#   base, same hybrid Qwen3_5 arch: mog-sec crashed five times (420k -> 384k -> 320k -> ...)
#   because every cut sized the KV POOL while the crashes were governed by PROCESSING DEPTH.
#   163840 buys a clean 400-refusal above the measured ceiling instead of an engine death.
#   Do NOT raise without re-running a deep-prefill probe with a NON-REPEATING prompt on THIS
#   checkpoint — a repeated prompt hashes to cached blocks and never prefills deep.
#
# ⚠ Speculative decoding: dflash k=7 is the DEFAULT here, carried over from the sentinel
#   measurement (dflash 2.40 vs MTP 2.18 mean acceptance length on a 27B body, +11%). But
#   cyberprev is an ABLITERATED body, not an SFT one, and abliteration is exactly what can
#   desync an MTP head — so the two are re-measured on THIS checkpoint before believing either.
#   See the CYBER_SPEC_CONFIG note below.
name: cyberprev-seat

services:
  vllm-cyberprev:
    image: ${CYBER_IMAGE:-vllm/vllm-openai:nightly-e9d1398d9edfd90fcc1cf783805240e3effec013}
    container_name: ${CYBER_CONTAINER_NAME:-vllm-cyberprev}
    restart: unless-stopped
    ipc: host
    ports:
      - "${CYBER_PORT:-8025}:8000"
    volumes:
      - /tank/aimodels/huggingface:/hfcache
      - ${CYBER_MODEL:-/tank/aimodels/cyberprev-nvfp4-mixed-v2}:/model:ro
      # DFlash2 drafter, mounted unconditionally — inert if CYBER_SPEC_CONFIG selects an MTP
      # method, which does not reference /drafter. Shared with mog-sec and (formerly)
      # sentinel-r3: all three share the same 64-layer Qwen3.8-27B base and vocab, so one
      # drafter serves them all. Costs +3.6 GB of VRAM when dflash is active.
      - ${CYBER_DRAFT_MODEL:-/tank/aimodels/qwen38-27b-dflash2-drafter}:/drafter:ro
    environment:
      - VLLM_API_KEY=${API_KEY:-}
      - PYTORCH_CUDA_ALLOC_CONF=${CYBER_ALLOC_CONF:-}
    command:
      - /model
      # ⚠ Served under its OWN name. It does NOT inherit `sentinel-r3` — serving different
      # weights under a retired model's name is silent substitution. The sentinel-r3 gateway
      # aliases are left to 404 deliberately; a `cyberprev` alias is added instead.
      - --served-model-name
      - ${CYBER_SERVED_NAME:-cyberprev-27b}
      - ${CYBER_SERVED_NAME_THINK:-cyberprev-27b-thinking}
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      - --quantization
      - ${CYBER_QUANT:-compressed-tensors}
      - --gpu-memory-utilization
      - "${CYBER_GPU_MEM_UTIL:-0.40}"
      # KV pinned in BYTES, not a ratio — a ratio silently yields a different cache depending
      # on what else is resident on GPU 0 at start time (mog-sec is co-resident here).
      - --kv-cache-memory
      - "${CYBER_KV_CACHE_MEMORY:-8589934592}"
      - --max-model-len
      - "${CYBER_MAX_MODEL_LEN:-163840}"
      - --max-num-seqs
      - "${CYBER_MAX_NUM_SEQS:-16}"
      - --max-num-batched-tokens
      - "${CYBER_MAX_NUM_BATCHED_TOKENS:-4096}"
      - --trust-remote-code
      - --dtype
      - auto
      # Hybrid backbone: linear_attention + full_attention layers; the model's own config
      # asks for float32 mamba/SSM state.
      - --mamba-cache-dtype
      - float32
      - --kv-cache-dtype
      - ${CYBER_KV_CACHE_DTYPE:-fp8}
      - --enable-prefix-caching
      - --enable-chunked-prefill
      - --limit-mm-per-prompt
      - '${CYBER_LIMIT_MM:-{"image": 4}}'
      - --mm-processor-kwargs
      - '${CYBER_MM_PROCESSOR_KWARGS:-{"size": {"longest_edge": 4194304, "shortest_edge": 65536}}}'
      - --reasoning-parser
      - ${CYBER_REASONING_PARSER:-qwen3}
      - --default-chat-template-kwargs
      - '{"reasoning_effort": "${CYBER_REASONING_EFFORT:-medium}"}'
      - --enable-auto-tool-choice
      - --tool-call-parser
      - qwen3_coder
      # ONE env var carrying the whole JSON, because the two speculative shapes are not
      # interchangeable: dflash needs a "model" pointing at the drafter, MTP must NOT have
      # one. A method+tokens template cannot express both.
      #   DFlash2 : {"method": "dflash", "model": "/drafter", "num_speculative_tokens": 7}
      #   MTP     : {"method": "qwen3_5_mtp", "num_speculative_tokens": 3}
      # ⚠ Comparing the two requires MATCHING num_speculative_tokens, or you are comparing
      # draft width as well as method — quant playbook §5.1: MTP runs a single-module head
      # autoregressively, so deeper k improves acceptance and destroys throughput.
      # ⚠ To turn speculation OFF there is no honest placeholder value — delete these two
      # lines rather than passing an empty string.
      - --speculative-config
      - '${CYBER_SPEC_CONFIG:-{"method": "dflash", "model": "/drafter", "num_speculative_tokens": 7}}'
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids:
                - "${CYBER_GPU_ID:-0}"
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 900s
    networks:
      - tnet
    labels:
      - homepage.group=AI - Inference
      - homepage.name=CyberPreview (abliterated sec)
      - homepage.icon=mdi-shield-bug
      - homepage.description=Cyber offensive-security tool-calling SFT (LoRA) on abliterated Qwen3.8-27B, sec seat (fv-ml1 GPU 0)
      - homepage.href=http://10.251.50.54:${CYBER_PORT:-8025}/docs

networks:
  tnet:
    name: traefik-net
    external: true