esh-pfi-infrastructure/stacks/sentinel-r3/compose.yaml

# sentinel-r3 — Sentinel-R3 pen-test seat, A/B candidate ALONGSIDE mog-sec on fv-ml1 GPU 0.
#
# glyphsoftware/sentinel-r3: a REAL SFT finetune of stock Qwen/Qwen3.8-27B on 1,230
# authorized-pentest agent trajectories (recon -> foothold -> privesc -> writeup) over a
# 19-tool surface that matches our own harness. Contrast mog-sec (M.O.G.-SEC/Qwentium),
# which is a refusal-free OFFENSE+DEFENSE cyber SFT of Qwen3.8-27B — its card explicitly says
# "not a system-prompt sticker on a stock Qwen" (an earlier note here called it that; wrong).
# So all three sec seats are Qwen3.8-27B finetunes, differing in training focus. Quantized in-house to the same mixed NVFP4 W4A4(MLP
# 0-55) + FP8 W8A8(attn/lm_head/MLP 56-63) recipe as mog-sec/gen. → sentinel-r3-nvfp4-mixed.PROVENANCE.txt
#
# ⚠ PROPRIETARY LICENSE (Glyph Proprietary v1.0) — operator's fair-use/licensee call, unlike
#   mog-sec's Apache. Served here on operator instruction 2026-09-14.
#
# ⚠ dflash speculative decoding ENABLED 2026-09-14 after measurement (see the spec-config note
#   below). Originally served without spec to establish a clean baseline; the probe measured
#   dflash k=7 at 2.40 acceptance length vs MTP k=3 at 2.18 on this finetuned body.
# (historical) SERVED WITHOUT --speculative-config ON PURPOSE. The MTP head is a VERBATIM base graft
#   (source shipped zero mtp.*), and its acceptance on this SFT-finetuned body is UNVERIFIED —
#   the gate is a measured >=~40% on a probe serve, not an assumption. A clean no-spec boot
#   establishes the quality baseline for the mog-sec A/B first; MTP acceptance is a separate
#   measurement (add the qwen3_5_mtp spec-config and read the acceptance metric).
#
# ⚠ max-model-len 163840, NOT native 262K. Sentinel is the SAME base + hybrid Qwen3_5 arch as
#   mog-sec, so the identical deep-context lesson applies: what the KV pool HOLDS and what the
#   card PROCESSES at depth are different numbers, and mog-sec crashed five times before 163840
#   bought a clean 400-refusal above the measured ceiling instead of an engine death. Do not
#   raise without re-running the deep-ctx probe on THIS checkpoint.
#
# ⚠ Serve with a PROSE system prompt — Sentinel was trained on prose tool descriptions, not
#   structured `tools=`.

name: sentinel-r3

services:
  vllm-sentinel-r3:
    image: ${SENTINEL_IMAGE:-vllm/vllm-openai:nightly-e9d1398d9edfd90fcc1cf783805240e3effec013}
    container_name: ${SENTINEL_CONTAINER_NAME:-vllm-sentinel-r3}
    restart: unless-stopped
    ipc: host
    ports:
      - "${SENTINEL_PORT:-8025}:8000"
    volumes:
      - /tank/aimodels/huggingface:/hfcache
      - ${SENTINEL_MODEL:-/tank/aimodels/sentinel-r3-nvfp4-mixed}:/model:ro
      - ${SENTINEL_DRAFT_MODEL:-/tank/aimodels/qwen38-27b-dflash2-drafter}:/drafter:ro
    environment:
      - VLLM_API_KEY=${API_KEY:-}
    command:
      - /model
      - --served-model-name
      - ${SENTINEL_SERVED_NAME:-sentinel-r3}
      - ${SENTINEL_SERVED_NAME_THINK:-sentinel-r3-thinking}
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      - --quantization
      - ${SENTINEL_QUANT:-compressed-tensors}
      - --gpu-memory-utilization
      - "${SENTINEL_GPU_MEM_UTIL:-0.40}"
      # KV pinned in bytes — same discipline as mog-sec/erp-seat: a ratio yields a different
      # cache depending on what else is resident at start, an explicit figure is reproducible.
      # 8 GiB is generous for an A/B probe (conc 1/4/8 short prompts never approach it).
      - --kv-cache-memory
      - "${SENTINEL_KV_CACHE_MEMORY:-8589934592}"
      - --max-model-len
      - "${SENTINEL_MAX_MODEL_LEN:-163840}"
      - --max-num-seqs
      - "${SENTINEL_MAX_NUM_SEQS:-16}"
      - --max-num-batched-tokens
      - "${SENTINEL_MAX_NUM_BATCHED_TOKENS:-4096}"
      - --trust-remote-code
      - --dtype
      - auto
      - --mamba-cache-dtype
      - float32
      - --kv-cache-dtype
      - ${SENTINEL_KV_CACHE_DTYPE:-fp8}
      - --enable-prefix-caching
      - --enable-chunked-prefill
      - --limit-mm-per-prompt
      - '${SENTINEL_LIMIT_MM:-{"image": 4}}'
      - --mm-processor-kwargs
      - '${SENTINEL_MM_PROCESSOR_KWARGS:-{"size": {"longest_edge": 4194304, "shortest_edge": 65536}}}'
      - --reasoning-parser
      - ${SENTINEL_REASONING_PARSER:-qwen3}
      - --default-chat-template-kwargs
      - '{"reasoning_effort": "${SENTINEL_REASONING_EFFORT:-medium}"}'
      - --enable-auto-tool-choice
      - --tool-call-parser
      - qwen3_coder
      # dflash speculative decoding — MEASURED 2026-09-14 on THIS finetuned body: dflash k=7
      # gives mean acceptance length 2.40 vs MTP k=3 at 2.18 (+11%, clean n=2 separation).
      # dflash wins on reach (accepts a longer tail to k=7) despite equal ~60% position-1
      # acceptance. The drafter is the same qwen38-27b-dflash2-drafter mog-sec uses (Sentinel
      # shares its 64-layer Qwen3.8-27B base). Costs +3.6 GB for the drafter.
      - --speculative-config
      - '${SENTINEL_SPEC_CONFIG:-{"method": "dflash", "model": "/drafter", "num_speculative_tokens": 7}}'
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids:
                - "${SENTINEL_GPU_ID:-0}"
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 900s
    networks:
      - tnet
    labels:
      - homepage.group=AI - Inference
      - homepage.name=Sentinel-R3 (pen-test A/B)
      - homepage.icon=mdi-shield-search
      - homepage.description=SFT pen-test finetune of Qwen3.8-27B, A/B candidate vs mog-sec (fv-ml1 GPU 0)
      - homepage.href=http://10.251.50.54:${SENTINEL_PORT:-8025}/docs

networks:
  tnet:
    name: traefik-net
    external: true