diff --git a/stacks/cyberprev-seat/.env.example b/stacks/cyberprev-seat/.env.example new file mode 100644 index 0000000..e16cc4a --- /dev/null +++ b/stacks/cyberprev-seat/.env.example @@ -0,0 +1,54 @@ +# cyberprev-seat — hotdogs/Qwen3.8-27B-abliterated-cyber-preview, fv-ml1 GPU 0, :8025. +# Copy to .env on the host at /opt/docker/compose/cyberprev-seat/.env. + +# ── Image ─────────────────────────────────────────────────────────────────── +# Same pinned nightly mog-sec and sentinel-r3 run. Not :latest — pin it, so a seat +# restart cannot silently change the engine under a measured configuration. +CYBER_IMAGE=vllm/vllm-openai:nightly-e9d1398d9edfd90fcc1cf783805240e3effec013 +API_KEY=replace-me + +# ── Placement ─────────────────────────────────────────────────────────────── +# GPU 0, co-resident with mog-sec. Took over :8025 from the retired sentinel-r3 seat. +CYBER_GPU_ID=0 +CYBER_PORT=8025 +CYBER_CONTAINER_NAME=vllm-cyberprev + +# ── Model ─────────────────────────────────────────────────────────────────── +# In-house mixed quant of /tank/aimodels/cyberprev-bf16 (51.0 GiB bf16 source), +# built with services/gen-seat-mixed-quant/. compressed-tensors, NOT modelopt_fp4. +CYBER_MODEL=/tank/aimodels/cyberprev-nvfp4-mixed +CYBER_QUANT=compressed-tensors +# ⚠ Its OWN name. Do not reuse `sentinel-r3` — that seat is retired and its gateway +# aliases are deliberately left to 404 rather than repointed at different weights. +CYBER_SERVED_NAME=cyberprev-27b +CYBER_SERVED_NAME_THINK=cyberprev-27b-thinking + +# ── Memory ────────────────────────────────────────────────────────────────── +# 0.40 of the card, sharing GPU 0 with mog-sec. KV pinned in bytes (8 GiB) so the +# figure is reproducible regardless of what else is resident at start time. +CYBER_GPU_MEM_UTIL=0.40 +CYBER_KV_CACHE_MEMORY=8589934592 + +# ⚠ 163840, NOT native 262144. Inherited from mog-sec, which crashed five times +# learning that the KV pool's capacity and the card's processing depth are different +# numbers. Raising this requires a deep-prefill probe with a NON-REPEATING prompt on +# THIS checkpoint — a repeated prompt hashes to cached blocks and never prefills deep. +CYBER_MAX_MODEL_LEN=163840 +CYBER_MAX_NUM_SEQS=16 +CYBER_MAX_NUM_BATCHED_TOKENS=4096 +CYBER_KV_CACHE_DTYPE=fp8 + +# ── Speculative decoding ──────────────────────────────────────────────────── +# One var, whole JSON — the two shapes are not interchangeable (dflash needs a +# "model", MTP must not have one). +# DFlash2 : {"method": "dflash", "model": "/drafter", "num_speculative_tokens": 7} +# MTP : {"method": "qwen3_5_mtp", "num_speculative_tokens": 3} +# dflash k=7 is the default, carried from the sentinel measurement (2.40 vs 2.18). +# ⚠ Re-measure on THIS body before trusting it: cyberprev is ABLITERATED, and +# abliteration is precisely what can desync an MTP head. Gate MTP on a measured +# acceptance of >=~40%, never on an assumption. +CYBER_SPEC_CONFIG={"method": "dflash", "model": "/drafter", "num_speculative_tokens": 7} + +CYBER_REASONING_PARSER=qwen3 +CYBER_REASONING_EFFORT=medium +CYBER_ALLOC_CONF= diff --git a/stacks/cyberprev-seat/compose.yaml b/stacks/cyberprev-seat/compose.yaml new file mode 100644 index 0000000..085c69d --- /dev/null +++ b/stacks/cyberprev-seat/compose.yaml @@ -0,0 +1,141 @@ +# cyberprev-seat — hotdogs/Qwen3.8-27B-abliterated-cyber-preview, fv-ml1 GPU 0, :8025. +# +# The THIRD sec-seat candidate, and the one that displaced sentinel-r3 (operator instruction +# 2026-09-14). Lineage of the three, because the distinction is the whole point of the A/B: +# mog-sec — persona system prompt on STOCK Qwen3.8-27B weights +# sentinel-r3 — a REAL SFT pentest finetune (proprietary licence); RETIRED for this seat +# cyberprev — an ABLITERATION of Qwen3.8-27B (refusal-direction weight edit, no finetune) +# aimed at the cyber-offense refusal surface specifically +# +# Quantized in-house to the same mixed recipe as mog-sec / sentinel / gen: +# NVFP4 W4A4 on MLP layers 0-55 + FP8 W8A8 on attn / linear_attn / lm_head / MLP 56-63, +# FP8 KV, vision tower + linear_attn norms + `re:^mtp.*` ignored. +# Pipeline: services/gen-seat-mixed-quant/. Source bf16: /tank/aimodels/cyberprev-bf16. +# +# ⚠ `re:^mtp.*` MUST be present in config.json quantization_config.ignore, or vLLM loads the +# grafted BF16 MTP head UNINITIALISED and speculative decoding silently runs at 0% +# acceptance. llm-compressor PRUNES ignore entries that matched no module at quant time, +# and the wrapper class never loads the MTP head — so the entry is dropped unless +# post_quant.py re-injects it. post_quant.py verifies rather than assumes; do not skip it. +# +# ⚠ max-model-len 163840, NOT the native 262K — inherited from mog-sec the hard way. Same +# base, same hybrid Qwen3_5 arch: mog-sec crashed five times (420k -> 384k -> 320k -> ...) +# because every cut sized the KV POOL while the crashes were governed by PROCESSING DEPTH. +# 163840 buys a clean 400-refusal above the measured ceiling instead of an engine death. +# Do NOT raise without re-running a deep-prefill probe with a NON-REPEATING prompt on THIS +# checkpoint — a repeated prompt hashes to cached blocks and never prefills deep. +# +# ⚠ Speculative decoding: dflash k=7 is the DEFAULT here, carried over from the sentinel +# measurement (dflash 2.40 vs MTP 2.18 mean acceptance length on a 27B body, +11%). But +# cyberprev is an ABLITERATED body, not an SFT one, and abliteration is exactly what can +# desync an MTP head — so the two are re-measured on THIS checkpoint before believing either. +# See the CYBER_SPEC_CONFIG note below. +name: cyberprev-seat + +services: + vllm-cyberprev: + image: ${CYBER_IMAGE:-vllm/vllm-openai:nightly-e9d1398d9edfd90fcc1cf783805240e3effec013} + container_name: ${CYBER_CONTAINER_NAME:-vllm-cyberprev} + restart: unless-stopped + ipc: host + ports: + - "${CYBER_PORT:-8025}:8000" + volumes: + - /tank/aimodels/huggingface:/hfcache + - ${CYBER_MODEL:-/tank/aimodels/cyberprev-nvfp4-mixed}:/model:ro + # DFlash2 drafter, mounted unconditionally — inert if CYBER_SPEC_CONFIG selects an MTP + # method, which does not reference /drafter. Shared with mog-sec and (formerly) + # sentinel-r3: all three share the same 64-layer Qwen3.8-27B base and vocab, so one + # drafter serves them all. Costs +3.6 GB of VRAM when dflash is active. + - ${CYBER_DRAFT_MODEL:-/tank/aimodels/qwen38-27b-dflash2-drafter}:/drafter:ro + environment: + - VLLM_API_KEY=${API_KEY:-} + - PYTORCH_CUDA_ALLOC_CONF=${CYBER_ALLOC_CONF:-} + command: + - /model + # ⚠ Served under its OWN name. It does NOT inherit `sentinel-r3` — serving different + # weights under a retired model's name is silent substitution. The sentinel-r3 gateway + # aliases are left to 404 deliberately; a `cyberprev` alias is added instead. + - --served-model-name + - ${CYBER_SERVED_NAME:-cyberprev-27b} + - ${CYBER_SERVED_NAME_THINK:-cyberprev-27b-thinking} + - --host + - 0.0.0.0 + - --port + - "8000" + - --quantization + - ${CYBER_QUANT:-compressed-tensors} + - --gpu-memory-utilization + - "${CYBER_GPU_MEM_UTIL:-0.40}" + # KV pinned in BYTES, not a ratio — a ratio silently yields a different cache depending + # on what else is resident on GPU 0 at start time (mog-sec is co-resident here). + - --kv-cache-memory + - "${CYBER_KV_CACHE_MEMORY:-8589934592}" + - --max-model-len + - "${CYBER_MAX_MODEL_LEN:-163840}" + - --max-num-seqs + - "${CYBER_MAX_NUM_SEQS:-16}" + - --max-num-batched-tokens + - "${CYBER_MAX_NUM_BATCHED_TOKENS:-4096}" + - --trust-remote-code + - --dtype + - auto + # Hybrid backbone: linear_attention + full_attention layers; the model's own config + # asks for float32 mamba/SSM state. + - --mamba-cache-dtype + - float32 + - --kv-cache-dtype + - ${CYBER_KV_CACHE_DTYPE:-fp8} + - --enable-prefix-caching + - --enable-chunked-prefill + - --limit-mm-per-prompt + - '${CYBER_LIMIT_MM:-{"image": 4}}' + - --mm-processor-kwargs + - '${CYBER_MM_PROCESSOR_KWARGS:-{"size": {"longest_edge": 4194304, "shortest_edge": 65536}}}' + - --reasoning-parser + - ${CYBER_REASONING_PARSER:-qwen3} + - --default-chat-template-kwargs + - '{"reasoning_effort": "${CYBER_REASONING_EFFORT:-medium}"}' + - --enable-auto-tool-choice + - --tool-call-parser + - qwen3_coder + # ONE env var carrying the whole JSON, because the two speculative shapes are not + # interchangeable: dflash needs a "model" pointing at the drafter, MTP must NOT have + # one. A method+tokens template cannot express both. + # DFlash2 : {"method": "dflash", "model": "/drafter", "num_speculative_tokens": 7} + # MTP : {"method": "qwen3_5_mtp", "num_speculative_tokens": 3} + # ⚠ Comparing the two requires MATCHING num_speculative_tokens, or you are comparing + # draft width as well as method — quant playbook §5.1: MTP runs a single-module head + # autoregressively, so deeper k improves acceptance and destroys throughput. + # ⚠ To turn speculation OFF there is no honest placeholder value — delete these two + # lines rather than passing an empty string. + - --speculative-config + - '${CYBER_SPEC_CONFIG:-{"method": "dflash", "model": "/drafter", "num_speculative_tokens": 7}}' + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${CYBER_GPU_ID:-0}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 900s + networks: + - tnet + labels: + - homepage.group=AI - Inference + - homepage.name=CyberPreview (abliterated sec) + - homepage.icon=mdi-shield-bug + - homepage.description=Qwen3.8-27B abliterated for cyber-offense refusals, sec-seat candidate (fv-ml1 GPU 0) + - homepage.href=http://10.251.50.54:${CYBER_PORT:-8025}/docs + +networks: + tnet: + name: traefik-net + external: true