diff --git a/servers/ana-ml2/README.md b/servers/ana-ml2/README.md index 208d489..db3bbcc 100644 --- a/servers/ana-ml2/README.md +++ b/servers/ana-ml2/README.md @@ -41,7 +41,7 @@ embed/rerank/reward trio. GPUs are pinned per container via | Container | Port | Served model | Quant | Ctx | |-----------|------|--------------|-------|-----| -| `vllm-aeon-gen` | 8015 | `qwen3.6-35b-a3b-heretic` — the "gen" hero seat | NVFP4 (modelopt) | 256k | +| `vllm-gen` (project `gen-seat`) | 8015 | `qwen3.8-27b-uncensored` — the "gen" hero seat (Qwen3.8-27B Heretic-abliterated, in-house NVFP4 W4A16 + grafted MTP) | NVFP4 W4A16 (compressed-tensors) | 262k | | `vllm-charrp-reasoning-nvfp4` | 8018 | `char-rp-reasoning` (R36 reasoning RP) | NVFP4 (modelopt) | 256k | **GPU 1 — light / eval / retrieval + char-RP GGUF (~91/98 GB, on-demand):** @@ -91,7 +91,7 @@ Latest snapshot: `system-details.txt` (regenerate as needed). Every seat is explicitly pinned via `device_ids` (no unpinned containers), and both cards run ~90% full: -- **GPU 0:** the two heavy NVFP4 seats — `vllm-aeon-gen` (gen) and +- **GPU 0:** the two heavy NVFP4 seats — `vllm-gen` (gen) and `vllm-charrp-reasoning-nvfp4`. The live serving path (near-100% util under load), ~42 + 45 GB. - **GPU 1:** everything else — summarizer (granite), judge (selene), reward, diff --git a/stacks/gen-seat/.env.example b/stacks/gen-seat/.env.example new file mode 100644 index 0000000..a19d2ef --- /dev/null +++ b/stacks/gen-seat/.env.example @@ -0,0 +1,18 @@ +# gen-seat tunables — fleet `gen` seat (ana-ml2 GPU 0, :8015). Edit here, never commit. +GEN_IMAGE=vllm/vllm-openai:latest +API_KEY= +GEN_GPU_ID=0 + +GEN_CONTAINER_NAME=vllm-gen +GEN_PORT=8015 +GEN_SERVED_NAME=qwen3.8-27b-uncensored +GEN_SERVED_NAME_THINK=qwen3.8-27b-uncensored-thinking +GEN_MODEL=/tank/aimodels/qwen38-27b-uncensored-nvfp4 +GEN_QUANT=compressed-tensors +GEN_GPU_MEM_UTIL=0.45 +GEN_MAX_MODEL_LEN=262144 +GEN_MAX_NUM_SEQS=16 +GEN_KV_CACHE_DTYPE=fp8 +GEN_REASONING_PARSER=qwen3 +GEN_SPEC_METHOD=qwen3_5_mtp +GEN_SPEC_TOKENS=3 diff --git a/stacks/gen-seat/compose.yaml b/stacks/gen-seat/compose.yaml new file mode 100644 index 0000000..32d0376 --- /dev/null +++ b/stacks/gen-seat/compose.yaml @@ -0,0 +1,97 @@ +# gen-seat — the fleet's general `gen` seat on ana-ml2 GPU 0 (:8015). +# Serves JonathanColetti/Qwen3.8-27B-Uncensored (Heretic abliteration KL 0.12, vision-intact +# Qwen3_5ForConditionalGeneration, grafted MTP head), quantized in-house to NVFP4 W4A16 +# (compressed-tensors) with the bf16 MTP grafted back. ⚠ the grafted MTP requires +# `re:^mtp.*` in config.json quantization_config.ignore or vLLM loads it uninitialized (0% accept). +# Backs gateway aliases: gen, gen-reasoning, summarizer, summarizer-large, classifier, +# image-judge, qwen-image-bench (all via api_base :8015). +# +# Two served-names (base + `-thinking`): LiteLLM keys deployments by (model, api_base), so gen +# and gen-reasoning use distinct names to avoid the shared-config enable_thinking clobber. +# --mamba-cache-dtype float32 for the GDN/hybrid-linear-attn state; MTP via qwen3_5_mtp n=3; +# full 262K context (hybrid attention -> cheap KV, ~8.6GB at 262K). All tunables in .env. +# (Renamed from qwen36-27b-aeon/vllm-aeon-gen; dormant RP service dropped 2026-08-15.) + +name: gen-seat + +services: + vllm-gen: + image: ${GEN_IMAGE:-vllm/vllm-openai:latest} + container_name: ${GEN_CONTAINER_NAME:-vllm-gen} + restart: unless-stopped + ipc: host + ports: + - "${GEN_PORT:-8015}:8000" + volumes: + - /tank/aimodels/huggingface:/hfcache + - ${GEN_MODEL:-/tank/aimodels/qwen38-27b-uncensored-nvfp4}:/model:ro + environment: + - HF_HOME=/hfcache + - HF_HUB_CACHE=/hfcache/hub + - VLLM_API_KEY=${API_KEY:-} + - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + command: + - /model + - --served-model-name + - ${GEN_SERVED_NAME:-qwen3.8-27b-uncensored} + - ${GEN_SERVED_NAME_THINK:-qwen3.8-27b-uncensored-thinking} + - --host + - 0.0.0.0 + - --port + - "8000" + - --quantization + - ${GEN_QUANT:-compressed-tensors} + - --gpu-memory-utilization + - ${GEN_GPU_MEM_UTIL:-0.45} + - --max-model-len + - ${GEN_MAX_MODEL_LEN:-262144} + - --max-num-seqs + - ${GEN_MAX_NUM_SEQS:-16} + - --max-num-batched-tokens + - "16384" + - --trust-remote-code + - --dtype + - auto + - --mamba-cache-dtype + - float32 + - --kv-cache-dtype + - ${GEN_KV_CACHE_DTYPE:-fp8} + - --enable-prefix-caching + - --enable-chunked-prefill + - --limit-mm-per-prompt + - '{"image": 4}' + - --reasoning-parser + - ${GEN_REASONING_PARSER:-qwen3} + - --enable-auto-tool-choice + - --tool-call-parser + - qwen3_coder + - --speculative-config + - '{"method": "${GEN_SPEC_METHOD:-qwen3_5_mtp}", "num_speculative_tokens": ${GEN_SPEC_TOKENS:-3}}' + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${GEN_GPU_ID:-0}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 900s + networks: + - tnet + labels: + - homepage.group=AI - Inference + - homepage.name=Qwen3.8-27B Uncensored (NVFP4, vision, MTP) — gen + - homepage.icon=mdi-creation + - homepage.description=Uncensored Qwen3.8-27B multimodal NVFP4+MTP, the `gen` seat (ana-ml2 GPU 0) + - homepage.href=http://10.250.50.54:${GEN_PORT:-8015}/docs + +networks: + tnet: + name: traefik-net + external: true diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 7b1ab2e..e2a9f98 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -22,44 +22,51 @@ model_list: # R15 P03 eval). vLLM on ana-ml2 GPU 1, official FP8, 50K ctx. Explicit # entry shadows the "*" wildcard's llama-swap route for this name. Full # prompt + completion captured per call. --- - - model_name: granite-4.1-8b - litellm_params: - model: hosted_vllm/granite-4.1-8b - api_base: http://10.250.50.54:8004/v1 - api_key: os.environ/VLLM_API_KEY - temperature: 0 - model_info: - mode: chat + # --- granite-4.1-8b RETIRED 2026-08-12 (seat downed, GPU1 reclaimed for RP context) --- +# - model_name: granite-4.1-8b +# litellm_params: +# model: hosted_vllm/granite-4.1-8b +# api_base: http://10.250.50.54:8004/v1 +# api_key: os.environ/VLLM_API_KEY +# temperature: 0 +# model_info: +# mode: chat - # alias: summarizer -> granite-4.1-8b (operator 2026-06-19). Duplicate-entry alias + # alias: summarizer -> gen (repointed 2026-08-12, granite retired) (operator 2026-06-19). Duplicate-entry alias # (not router_settings.model_group_alias — that's hidden from /v1/models and can be # silently ignored in config per litellm #15020/#5524). Keep api_base in sync above. - model_name: summarizer litellm_params: - model: hosted_vllm/granite-4.1-8b - api_base: http://10.250.50.54:8004/v1 + model: hosted_vllm/qwen3.8-27b-uncensored + api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY temperature: 0 + extra_body: + chat_template_kwargs: + enable_thinking: false model_info: mode: chat - # alias: classifier -> granite-4.1-8b (operator 2026-06-19). Light/fast classification + # alias: classifier -> gen (repointed 2026-08-12, granite retired) (operator 2026-06-19). Light/fast classification # + triage endpoint; same backend as summarizer. Keep api_base in sync above. - model_name: classifier litellm_params: - model: hosted_vllm/granite-4.1-8b - api_base: http://10.250.50.54:8004/v1 + model: hosted_vllm/qwen3.8-27b-uncensored + api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY temperature: 0 + extra_body: + chat_template_kwargs: + enable_thinking: false model_info: mode: chat - # alias: summarizer-large -> gen / qwen3.6-35b-a3b-heretic (operator 2026-07-05). For heavier + # alias: summarizer-large -> gen / qwen3.8-27b-uncensored (operator 2026-07-05). For heavier # summarization that wants the 35B-A3B heretic `gen` model instead of granite-8b. Thinking OFF # (matches gen). Keep api_base (:8015) + enable_thinking in sync with the gen record below. - model_name: summarizer-large litellm_params: - model: hosted_vllm/qwen3.6-35b-a3b-heretic + model: hosted_vllm/qwen3.8-27b-uncensored api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.5 @@ -75,14 +82,14 @@ model_list: # --- image-judge / qwen-image-bench — T2I quality JUDGE aliases. The dedicated # Qwen-Image-Bench NVFP4 backend (ana-ml2 GPU 1, :8014) was RETIRED 2026-07-15 # (operator: reclaim ~30GB GPU1) after the arbo→gen hero-judge switch. Both - # aliases now REPOINT to the gen backend (:8015, qwen3.6-35b-a3b-heretic, + # aliases now REPOINT to the gen backend (:8015, qwen3.8-27b-uncensored, # vision-intact), held at deterministic judge sampling (temp 0 / top_k 1) with # enable_thinking:false (a reasoning preamble breaks json_object). Revert = # `docker compose start` stacks/qwen-image-bench on ana-ml2 + repoint api_base # back to :8014 + model hosted_vllm/qwen-image-bench. --- - model_name: qwen-image-bench litellm_params: - model: hosted_vllm/qwen3.6-35b-a3b-heretic + model: hosted_vllm/qwen3.8-27b-uncensored api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY temperature: 0 @@ -96,7 +103,7 @@ model_list: mode: chat - model_name: image-judge litellm_params: - model: hosted_vllm/qwen3.6-35b-a3b-heretic + model: hosted_vllm/qwen3.8-27b-uncensored api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY temperature: 0 @@ -114,7 +121,7 @@ model_list: # Displaced AEON-27B 2026-07-08 (which had displaced qwopus3.5-122b 2026-07-05). MoE 35B-A3B # (256 experts / 8 active), qwen3_5_moe GDN-hybrid, native MTP preserved but served MTP-OFF # (spec-decode hurts concurrent aggregate). Served on :8015 via vLLM, - # served-name qwen3.6-35b-a3b-heretic. Thinking split = chat_template_kwargs.enable_thinking + + # served-name qwen3.8-27b-uncensored. Thinking split = chat_template_kwargs.enable_thinking + # --reasoning-parser qwen3; tool-calling qwen3_coder. gen / gen-reasoning + summarizer- # large route here; -reasoning enables thinking. Keep api_base (:8015) in sync. # RETIRED with the displacement (→ 404, callers migrate to gen): qwen3.5-122-a10b @@ -126,7 +133,7 @@ model_list: # GENERAL thinking profile, not the 0.6 coding sub-profile). docs/pfi/model-sampler-defaults.md. --- - model_name: gen litellm_params: - model: hosted_vllm/qwen3.6-35b-a3b-heretic + model: hosted_vllm/qwen3.8-27b-uncensored api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.5 @@ -142,7 +149,7 @@ model_list: litellm_params: # Distinct served-name so a thinking-off `gen` request can't mutate this deployment's # enable_thinking (shared-config-mutation footgun). Same backend :8015, different model id. - model: hosted_vllm/qwen3.6-35b-a3b-heretic-thinking + model: hosted_vllm/qwen3.8-27b-uncensored-thinking api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.5 @@ -167,7 +174,7 @@ model_list: # docs/pfi/model-sampler-defaults.md. - model_name: char-rp litellm_params: - model: hosted_vllm/magidonia-24b-v4.3 + model: hosted_vllm/char-rp api_base: http://10.250.50.54:8016/v1 api_key: os.environ/VLLM_API_KEY temperature: 1.1 @@ -248,9 +255,7 @@ model_list: top_p: 0.9 model_info: mode: chat - # reranker → generic capability name for rerank. Repointed to bge-reranker-v2-m3 - # 2026-08-06 (Brokkr R43: incumbent Qwen3-Reranker-0.6B harmed 80/90 fleet queries; - # boundary 17:37:48Z). Incumbent stays reachable via the qwen3-reranker alias for rollback. + # reranker → generic capability name for rerank (currently qwen3-reranker). - model_name: reranker litellm_params: model: hosted_vllm/BAAI/bge-reranker-v2-m3 diff --git a/stacks/qwen36-27b-aeon/.env.example b/stacks/qwen36-27b-aeon/.env.example deleted file mode 100644 index 82a59b0..0000000 --- a/stacks/qwen36-27b-aeon/.env.example +++ /dev/null @@ -1,53 +0,0 @@ -# qwen36-27b-aeon tunables — copy to .env on the host, edit there (never commit .env). -# The real .env lives at /opt/docker/compose/qwen36-27b-aeon/.env on ana-ml2. - -# Per-service vLLM image. Both on latest (0.24.0) after validating it on rp. Reasoning works -# on 0.24.0 (and did on 0.23.0 too — the trace bug was a LiteLLM shared-config mutation, not -# vLLM). Rollback: set either back to vllm/vllm-openai:known-good-0.23.0 (tagged on the host). -AEON_GEN_IMAGE=vllm/vllm-openai:latest -AEON_RP_IMAGE=vllm/vllm-openai:latest - -# Shared vLLM API key (matches the litellm VLLM_API_KEY). Blank = no auth. -API_KEY= - -# Both instances pin to the same physical GPU (the freed qwopus slot). -AEON_GPU_ID=0 - -# ── General serve (MTP off) → gen / gen-reasoning / summarizer-large ── -AEON_GEN_CONTAINER_NAME=vllm-aeon-gen -AEON_GEN_PORT=8015 -AEON_GEN_SERVED_NAME=qwen3.6-27b-aeon -AEON_GEN_SERVED_NAME_THINK=qwen3.6-27b-aeon-thinking -AEON_GEN_MODEL=/tank/aimodels/qwen36-27b-aeon-nvfp4 -# NOTE: live gen serves qwen3.6-35b-a3b-heretic since 2026-07-08 (aeon-27b displaced); -# SERVED_NAME/MODEL above lag live — pending reconciliation with the model swap. -# util/ctx/seqs RE-TUNED 2026-07-16 (char-rp left GPU-0 → room): util 0.30->0.42, -# ctx 131072->262144 (256K native), seqs 16->32. gen MoE KV is cheap → 5.43x @ 256K. -AEON_GEN_GPU_MEM_UTIL=0.42 -AEON_GEN_MAX_MODEL_LEN=262144 -AEON_GEN_MAX_NUM_SEQS=32 -AEON_GEN_KV_CACHE_DTYPE=fp8 -# Reasoning parser — KEEP qwen3. This checkpoint's chat_template.jinja injects the opening -# into the PROMPT (output has only ). qwen3 handles the non-thinking path -# correctly. NOTE: enable_thinking:true does NOT surface reasoning_content under EITHER qwen3 -# or deepseek_r1 (the reasoning is generated but dropped) — that needs a chat_template fix, -# NOT a parser swap. deepseek_r1 was tested and is WORSE: it breaks the non-thinking path -# too (both content and reasoning_content come back empty). Do not use deepseek_r1 here. -AEON_GEN_REASONING_PARSER=qwen3 - -# ── RP seat (native MTP on) → char-rp ── -AEON_RP_CONTAINER_NAME=vllm-aeon-rp -AEON_RP_PORT=8016 -AEON_RP_SERVED_NAME=qwen3.6-27b-aeon-rp -AEON_RP_SERVED_NAME_THINK=qwen3.6-27b-aeon-rp-thinking -# Full 27GB variant by default; switch to the 21GB XS for ~6GB more co-location margin: -# AEON_RP_MODEL=/tank/aimodels/qwen36-27b-aeon-nvfp4-xs -AEON_RP_MODEL=/tank/aimodels/qwen36-27b-aeon-nvfp4 -AEON_RP_GPU_MEM_UTIL=0.40 -AEON_RP_MAX_MODEL_LEN=65536 -AEON_RP_MAX_NUM_SEQS=2 -AEON_RP_KV_CACHE_DTYPE=fp8 -AEON_RP_REASONING_PARSER=qwen3 -# Native MTP head. If stock vLLM names it differently, try method=mtp. -AEON_RP_SPEC_METHOD=qwen3_5_mtp -AEON_RP_SPEC_TOKENS=3 diff --git a/stacks/qwen36-27b-aeon/compose.yaml b/stacks/qwen36-27b-aeon/compose.yaml deleted file mode 100644 index 821028a..0000000 --- a/stacks/qwen36-27b-aeon/compose.yaml +++ /dev/null @@ -1,214 +0,0 @@ -# qwen36-27b-aeon — AEON-7/Qwen3.6-27B-AEON-Ultimate-Uncensored on ana-ml2 GPU 0, -# REPLACING qwopus3.5-122b as the `gen` model (operator 2026-07-05). -# -# Dense 27B, qwen3_5 GDN-hybrid arch (full-attn + Gated DeltaNet SSM) — same family -# as qwopus, VISION-INTACT (Qwen3_5ForConditionalGeneration, vision tower preserved -# at bf16), abliterated (abliterix v1.4, 0/100 refusals), native MTP head grafted, -# Apache-2.0, 131K default ctx. Served NVFP4 (ModelOpt) on Blackwell's FP4 cores. -# -# TWO CO-LOCATED INSTANCES on GPU 0 (operator wants both behaviours resident at once; -# MTP is a serve-time config, NOT per-request, so one endpoint can't do both): -# vllm-aeon-gen (:8015, served qwen3.6-27b-aeon) — MTP OFF, general/concurrent -# serve. Backs gateway gen / gen-reasoning / summarizer-large. -# vllm-aeon-rp (:8016, served qwen3.6-27b-aeon-rp) — native MTP ON (qwen3_5_mtp -# n=3), low-concurrency single-seat RP. Backs gateway char-rp. -# Why MTP off for the general serve: measured on qwopus, MTP helps single-stream -# (+12% N=1) but HURTS moderate concurrency (-15..-20% N=4) and silently drops -# min_p/logit_bias — wrong for a shared multi-consumer endpoint. Right only for a -# dedicated single-stream seat (the RP one). [[reference_gen_qwopus_122b]] -# -# VRAM budget (2 weight copies, no sharing): full NVFP4 = 27GB ea. gen util 0.45 -# (~43GB) + rp util 0.40 (~38GB) = ~81GB / 96GB, ~15GB margin. depends_on: -# service_healthy sequences gen-first so the util reservation doesn't race → OOM. -# If margin bites at warmup (vision-encoder + big-vocab sampler warmup, cf. qwen36-vl), -# point AEON_RP_MODEL at the 21GB XS variant (frees ~6GB) via .env — no compose edit. -# -# NVFP4 is ModelOpt format → --quantization modelopt (vLLM also auto-detects; explicit -# is belt-and-suspenders). --mamba-cache-dtype float32 for the GDN/SSM state (AEON -# deploy guide + vLLM recipe Mamba-cache note). Tool-calling qwen3_coder + reasoning -# qwen3 (per the AEON card), same as qwopus. -# -# ⚠️ DEPLOYABILITY — load-test before trusting: multimodal + ModelOpt-NVFP4 on THIS -# brand-new arch, and native qwen3_5_mtp spec-decode, are unproven on our stock vLLM -# image. If stock can't serve it, the AEON patched image (ghcr.io/aeon-7/aeon-vllm- -# ultimate, PRs #41703/#40898) is the fallback — but that's really for DFlash; native -# MTP + base inference should ride stock >= 0.23.0. Set AEON_IMAGE in .env. -# -# REVERT: `docker compose down` here + `docker compose up -d` the qwopus3.5-122b stack -# (still staged) + revert the litellm gen/gen-reasoning/summarizer-large records. -# All tunables in .env — edit that, not this file. - -name: qwen36-27b-aeon - -services: - # ── General serve — MTP OFF, concurrent. gen / gen-reasoning / summarizer-large. ── - vllm-aeon-gen: - # Per-service image so gen can stay pinned to a known-good vLLM while rp tests a new one. - image: ${AEON_GEN_IMAGE:-vllm/vllm-openai:latest} - container_name: ${AEON_GEN_CONTAINER_NAME:-vllm-aeon-gen} - restart: unless-stopped - ipc: host - ports: - - "${AEON_GEN_PORT:-8015}:8000" - volumes: - - /tank/aimodels/huggingface:/hfcache - - ${AEON_GEN_MODEL:-/tank/aimodels/qwen36-27b-aeon-nvfp4}:/model:ro - environment: - - HF_HOME=/hfcache - - HF_HUB_CACHE=/hfcache/hub - - VLLM_API_KEY=${API_KEY:-} - # Reclaims PyTorch reserved-but-unallocated fragmentation so the co-located - # util split doesn't strand VRAM (same knob qwopus needed for the MoE workspace). - - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True - command: - - /model - # TWO served-names: base + a `-thinking` alias. LiteLLM keys deployments by - # (model, api_base), so gen and gen-reasoning MUST use distinct model names or a - # thinking-off request mutates the shared litellm_params and clobbers the other's - # enable_thinking (the shared-config-mutation footgun). gen-reasoning routes to the - # `-thinking` name; gen/summarizer-large route to the base name. - - --served-model-name - - ${AEON_GEN_SERVED_NAME:-qwen3.6-27b-aeon} - - ${AEON_GEN_SERVED_NAME_THINK:-qwen3.6-27b-aeon-thinking} - - --host - - 0.0.0.0 - - --port - - "8000" - - --quantization - - modelopt - - --gpu-memory-utilization - - ${AEON_GEN_GPU_MEM_UTIL:-0.45} - - --max-model-len - - ${AEON_GEN_MAX_MODEL_LEN:-131072} - # Keep concurrency modest: big-vocab sampler warmup allocates a large tensor - # (qwen36-vl OOM'd at the default 1024 on a shared GPU). 16 is ample here. - - --max-num-seqs - - ${AEON_GEN_MAX_NUM_SEQS:-16} - - --max-num-batched-tokens - - "16384" - - --trust-remote-code - - --dtype - - auto - # GDN/SSM (Gated DeltaNet) state cache — float32 per the AEON deploy guide. - - --mamba-cache-dtype - - float32 - - --kv-cache-dtype - - ${AEON_GEN_KV_CACHE_DTYPE:-fp8} - - --enable-prefix-caching - - --enable-chunked-prefill - - --limit-mm-per-prompt - - '{"image": 4}' - - --reasoning-parser - - ${AEON_GEN_REASONING_PARSER:-qwen3} - - --enable-auto-tool-choice - - --tool-call-parser - - qwen3_coder - deploy: - resources: - reservations: - devices: - - driver: nvidia - device_ids: - - "${AEON_GPU_ID:-0}" - capabilities: - - gpu - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 900s - networks: - - tnet - labels: - - homepage.group=AI - Inference - - homepage.name=Qwen3.6-27B AEON (NVFP4, vision) — gen - - homepage.icon=mdi-creation - - homepage.description=Uncensored Qwen3.6-27B multimodal NVFP4, the `gen` model (ana-ml2 GPU 0) - - homepage.href=http://10.250.50.54:${AEON_GEN_PORT:-8015}/docs - - # ── RP seat — native MTP ON, low concurrency, single-seat. char-rp. ── - vllm-aeon-rp: - image: ${AEON_RP_IMAGE:-vllm/vllm-openai:latest} - container_name: ${AEON_RP_CONTAINER_NAME:-vllm-aeon-rp} - restart: unless-stopped - ipc: host - # Sequence AFTER the general serve is healthy so the two util reservations on the - # shared GPU don't race into an OOM (gen reserves its 0.45 first, then rp its 0.40). - depends_on: - vllm-aeon-gen: - condition: service_healthy - ports: - - "${AEON_RP_PORT:-8016}:8000" - volumes: - - /tank/aimodels/huggingface:/hfcache - # Point at the XS (21GB) variant via .env to buy ~6GB co-location margin. - - ${AEON_RP_MODEL:-/tank/aimodels/qwen36-27b-aeon-nvfp4}:/model:ro - environment: - - HF_HOME=/hfcache - - HF_HUB_CACHE=/hfcache/hub - - VLLM_API_KEY=${API_KEY:-} - - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True - command: - - /model - # base + `-thinking` alias (see the gen note): char-rp -> base, char-rp-reasoning -> -thinking. - - --served-model-name - - ${AEON_RP_SERVED_NAME:-qwen3.6-27b-aeon-rp} - - ${AEON_RP_SERVED_NAME_THINK:-qwen3.6-27b-aeon-rp-thinking} - - --host - - 0.0.0.0 - - --port - - "8000" - - --quantization - - modelopt - - --gpu-memory-utilization - - ${AEON_RP_GPU_MEM_UTIL:-0.40} - - --max-model-len - - ${AEON_RP_MAX_MODEL_LEN:-65536} - # Single-seat: low concurrency keeps warmup + KV small so it fits alongside gen. - - --max-num-seqs - - ${AEON_RP_MAX_NUM_SEQS:-2} - - --trust-remote-code - - --dtype - - auto - - --mamba-cache-dtype - - float32 - - --kv-cache-dtype - - ${AEON_RP_KV_CACHE_DTYPE:-fp8} - - --enable-prefix-caching - - --limit-mm-per-prompt - - '{"image": 4}' - - --reasoning-parser - - ${AEON_RP_REASONING_PARSER:-qwen3} - # Native MTP speculative decode (the grafted head). n=3 per the AEON card's - # measured accept length (~3.3/3). MTP is why this seat exists separately. - - --speculative-config - - '{"method": "${AEON_RP_SPEC_METHOD:-qwen3_5_mtp}", "num_speculative_tokens": ${AEON_RP_SPEC_TOKENS:-3}}' - deploy: - resources: - reservations: - devices: - - driver: nvidia - device_ids: - - "${AEON_GPU_ID:-0}" - capabilities: - - gpu - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 900s - networks: - - tnet - labels: - - homepage.group=AI - Dormant - - homepage.name=Qwen3.6-27B AEON RP (NVFP4 + MTP) — char-rp - - homepage.icon=mdi-drama-masks - - homepage.description=Uncensored Qwen3.6-27B, native MTP single-seat RP (char-rp), ana-ml2 GPU 0 - - homepage.href=http://10.250.50.54:${AEON_RP_PORT:-8016}/docs - -networks: - tnet: - name: traefik-net - external: true