esh-pfi-infrastructure/stacks/flash-next-seat/compose.yaml

# flash-next-seat — Qwen3.8-Flash-Next (abliterated) on fv-ml1 GPU 2 (:8022).
#
# THE POINT OF THIS SEAT: it is the first model on the fleet whose weights do NOT
# fit the card, and it runs anyway. Qwen3.8-Flash-Next is 176B total — a 125B main
# model plus a 51B n-gram (PLE) lookup table — at ~6B active per token. The n-gram
# table is a pure lookup with almost no compute, so it lives in PINNED HOST RAM and
# the GPU reads the rows it needs directly over CUDA UVA. Resident on the card:
# ~78 GiB of 95.6. Resident in host RAM: 47.7 GiB of 566 GB. See --engram-config.
#
# Serves dealignai/Qwen3.8-Flash-Next-ABLITERATED-NVFP4, revision
# be794b990578ef3031eccf9f28e675a289a09ee9, pulled to
# /tank/aimodels/qwen38-flash-next-abliterated-nvfp4 (126.0 GiB on disk).
#   - routed experts (294,912 tensors, 48 layers x 512 experts) NVFP4 W4A4
#   - PLE n-gram table FP8 E4M3, 10 `model-plefp8-*` shards + per-table scalar scale,
#     taken from Qwen/Qwen3.8-Flash-Next-FP8 rather than re-derived
#   - everything else source precision: self_attn, linear_attn, mlp.gate,
#     shared_expert, hyper_connection, ple projections, visual tower, embeddings,
#     lm_head and all 31 MTP tensors are in the quant config's `ignore` list
#   - abliteration is a direct weight edit, no fine-tune: HarmBench 240-real-harm
#     compliance 100% at reasoning off/low/xhigh, MMLU 82.11 -> 81.93 (-0.18pp),
#     GSM8K 97.27% (full 1319), AIME26 pass@1 98.75% (30 x 8 repeats, SEM 0.61pp).
#     Publisher's own reports are kept in the model dir; they state protocols and
#     repeat counts, which is why this build was chosen over better-liked ones.
#
# ⚠ THE CHECKPOINT-SELECTION TRAP, recorded because it silently disqualifies most
# community builds. vLLM picks the PLE table's format from
# `text_config.ple_embedding_dtype` as the FIRST branch of
# Qwen4ExpPLEEmbeddingMethod.from_quant_config. This checkpoint declares
# "float8_e4m3fn". A checkpoint with an FP8 PLE table and NO such declaration
# (e.g. gorbatjovy/qwen3.8-flash-next-abliterated-NVFP4-plefp8) resolves instead
# through ModelOpt's `*.ple.*` exclude to the UNQUANTIZED method and fails to load
# on the scale tensor it does ship. Check that field before trying another build.
#
# ⚠ IMAGE: must contain vLLM PR #54371 ("UVA PLE-offload and Engram tensor
# parallelism", merged 2026-09-09T14:32Z). Verified by ancestry, not version string:
# nightly-eed1f3d0... is +150 commits / behind_by=0 from merge commit 3116c5d0.
# NOT in v0.29.0 — that release was cut ~6h before the merge. The older worker-based
# offload (#53899) is PAUSED upstream in favour of this one; do not go back to it,
# its entire bug family (TP=1 deadlock, pidfd_getfd/yama ptrace gate, shared-event
# race, silently one-step-stale outputs under CUDA graphs) came from the separate
# worker process that the UVA path does not have.
#
# ⚠⚠ MTP IS DELIBERATELY OFF, against our house graft-MTP habit. The model ships a
# working MTP head and this checkpoint preserves it byte-identically (31 tensors,
# audit passed) — we still do not enable it. vLLM's own recipe measured MTP on
# 4xH100 as WORSE AT EVERY CONCURRENCY tested: 8-36% lower request throughput,
# 32-173% higher per-token latency, ~36% acceptance, and says do not default it on.
# Open #55357 reports episodic 0% draft acceptance with repetition collapse inside
# thinking blocks. If you want it, it is FN_SPEC_CONFIG — but measure it here, with
# repeats, before believing it.
name: flash-next-seat
services:
  vllm-flash-next:
    image: ${FN_IMAGE:-vllm/vllm-openai:latest}
    container_name: ${FN_CONTAINER_NAME:-vllm-flash-next}
    restart: unless-stopped
    ipc: host
    # The PLE table is 47.7 GiB of PINNED host memory. Pinned pages are not
    # swappable and count against the process's locked-memory limit, so lift it.
    ulimits:
      memlock: -1
    ports:
      - "${FN_PORT:-8022}:8000"
    volumes:
      - /tank/aimodels/huggingface:/hfcache
      - ${FN_MODEL:-/tank/aimodels/qwen38-flash-next-abliterated-nvfp4}:/model:ro
    environment:
      - HF_HOME=/hfcache
      - HF_HUB_CACHE=/hfcache/hub
      - VLLM_API_KEY=${API_KEY:-}
      # Intentionally UNSET by default. expandable_segments has corrupted retained
      # tensors elsewhere on this box (quant playbook §3.10) and the offload path
      # hands UVA a pinned host allocation — not a combination anyone has tested.
      - PYTORCH_CUDA_ALLOC_CONF=${FN_ALLOC_CONF:-}
    command:
      - /model
      - --served-model-name
      - ${FN_SERVED_NAME:-qwen3.8-flash-next-uncensored}
      - ${FN_SERVED_NAME_THINK:-qwen3.8-flash-next-uncensored-thinking}
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      # ⚠ modelopt_fp4, NOT compressed-tensors as on the other seats here. This
      # checkpoint's quantization_config carries compressed-tensors-shaped
      # `config_groups` AND `quant_method: modelopt` / `quant_algo: NVFP4`
      # (producer nvidia-modelopt 0.46.0). The ModelOpt reader is the one whose
      # per-prefix `is_layer_excluded` honours the 13-entry ignore list that keeps
      # attention, the shared experts, the PLE and MTP out of W4A4.
      - --quantization
      - ${FN_QUANT:-modelopt_fp4}
      # ── THE WHOLE REASON THIS SEAT FITS ──────────────────────────────────────
      # Keeps the 51B n-gram table in pinned host RAM, read by the GPU over UVA on
      # a dedicated stream with async prefetch. Legacy VLLM_PLE_CPU_OFFLOAD=1 still
      # works but is deprecated in favour of this flag.
      # `embedding_across_dp` is irrelevant at TP=1/DP=1 — left at its default.
      - --engram-config
      - '${FN_ENGRAM_CONFIG:-{"cpu_offload": true}}'
      # GPU 2 is DEDICATED to this seat — nothing else is resident on the card, so
      # unlike gen/mog-sec on GPU 0/1 this ratio is not competing with a neighbour.
      # 0.90 is the vLLM recipe's own figure for offload configurations.
      # ⚠ PIN --kv-cache-memory IN BYTES once the first boot reports its budget
      # line. Same lesson as stacks/mog-sec and stacks/erp-seat: a ratio sizes the
      # KV calculation but does not bound peak activation, and it silently yields a
      # different cache depending on what is resident at start time.
      - --gpu-memory-utilization
      - ${FN_GPU_MEM_UTIL:-0.90}
      # ⚠ 128K TO START, NOT the native 262,144 the checkpoint advertises, and the
      # gap is deliberate. Two open upstream issues make depth the risky axis here:
      # #54764 (PLE short-conv batched prefill pads every request to the batch-max
      # query length, so one deep request inflates the whole step's activation) and
      # #54919 (long prefill starving active decode for 3-7 minutes). The mog-sec
      # lesson applies directly: what the KV pool can HOLD and what the card can
      # PROCESS at depth are different numbers, and sizing to the pool never fixed
      # a depth-driven crash. RAISE THIS ONLY AFTER BISECTING with a NON-REPEATING
      # prompt — a repeated one hashes to cached blocks and never prefills deep.
      - --max-model-len
      - ${FN_MAX_MODEL_LEN:-131072}
      - --max-num-seqs
      - ${FN_MAX_NUM_SEQS:-16}
      # 8192 rather than the 16384 used elsewhere, for the #54764 padding reason
      # above: a chunk's activation cost here scales with the batch's DEEPEST
      # request, not its average. Costs prefill throughput, costs no context.
      - --max-num-batched-tokens
      - ${FN_MAX_NUM_BATCHED_TOKENS:-8192}
      - --trust-remote-code
      - --dtype
      - auto
      # Hybrid backbone: 36 of 48 layers are Gated DeltaNet (linear attention), the
      # other 12 are Qwen Sparse Attention. The model's own config asks for
      # mamba_ssm_dtype float32.
      - --mamba-cache-dtype
      - float32
      # ⚠ MANDATORY-ISH: Qwen4Exp RAISES NotImplementedError on mamba_cache_mode
      # "all" and tells you to use "align". "align" is already the implicit default
      # when prefix caching is on, but it is stated here so that turning prefix
      # caching off cannot silently select a mode the model rejects.
      - --mamba-cache-mode
      - ${FN_MAMBA_CACHE_MODE:-align}
      # ⚠ FIRST THING TO TURN OFF if you see CUBLAS_STATUS_INTERNAL_ERROR or an
      # illegal memory access: open #54173 is exactly that, in the GDN path, WITH
      # prefix caching. On by default because every vLLM recipe config for this
      # model enables it.
      - ${FN_PREFIX_CACHING:---enable-prefix-caching}
      - --enable-chunked-prefill
      # ⚠ KV CACHE LEFT AT AUTO — do NOT copy `--kv-cache-dtype fp8` from the gen or
      # mog-sec seats. fp8_e4m3 KV on this model's QSA path is an unmerged RFC
      # (#54426: working patch, one machine, author asking for corroboration).
      #
      # Recipe sets this in every configuration; autotune has no value here and
      # costs startup time.
      - --no-enable-flashinfer-autotune
      - --limit-mm-per-prompt
      - '${FN_LIMIT_MM:-{"image": 4}}'
      # ⚠ MANDATORY. This checkpoint's preprocessor_config.json declares
      # size.longest_edge = 16777216 px (4096x4096), which expands to ~16,384 image
      # tokens — one image eating 12% of a 128K context, and enough to fail startup
      # on builds that enforce the text-vs-ids token count check. Same trap already
      # documented on stacks/mog-sec. 4194304 = 2048x2048 -> ~5,125 tokens.
      - --mm-processor-kwargs
      - '${FN_MM_PROCESSOR_KWARGS:-{"size": {"longest_edge": 4194304, "shortest_edge": 65536}}}'
      - --reasoning-parser
      - ${FN_REASONING_PARSER:-qwen3}
      # The Qwen3.8 chat template defaults to `xhigh`, where reasoning length grows
      # with conversation depth and has a long tail — measured on the 27B sibling at
      # 4,529-5,532 chars spiking to 9,261. `medium` is the seat-level default;
      # per-request override via chat_template_kwargs still works.
      - --default-chat-template-kwargs
      - '{"reasoning_effort": "${FN_REASONING_EFFORT:-medium}"}'
      - --enable-auto-tool-choice
      # qwen3_xml, per the vLLM recipe for THIS model — not the qwen3_coder parser
      # the mog-sec seat uses.
      - --tool-call-parser
      - ${FN_TOOL_CALL_PARSER:-qwen3_xml}
      # ── NO --speculative-config LINE, ON PURPOSE ─────────────────────────────
      # Speculative decoding is absent rather than disabled-by-a-flag, because
      # there is no honest "off" value for --speculative-config and a placeholder
      # flag in this slot is the kind of thing that gets copied forward and
      # misread. To turn MTP on, add these two lines here and nothing else:
      #     - --speculative-config
      #     - '{"method": "mtp", "num_speculative_tokens": 3}'
      # Read the MTP warning in the header first; the measured default is off.
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids:
                - "${FN_GPU_ID:-2}"
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      # ⚠ LONG. Cold boot reads 126 GiB from /tank and pins 47.7 GiB of it; the
      # closest published single-card timing is ~8.5 min load on a 96 GB card.
      # 1800s so a slow first boot is not mistaken for a failed one.
      start_period: 1800s
      retries: 3
    networks:
      - tnet
    labels:
      - homepage.group=AI - Inference
      - homepage.name=Qwen3.8-Flash-Next (uncensored)
      - homepage.icon=mdi-lightning-bolt-outline
      - homepage.description=176B/6B-active MoE, abliterated, NVFP4 + 48GB n-gram table offloaded to host RAM — fv-ml1 GPU 2
      - homepage.href=http://10.251.50.54:${FN_PORT:-8022}/docs
networks:
  tnet:
    name: traefik-net
    external: true