diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index 2ec3f38..a41ed22 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -54,13 +54,13 @@ model_list: model_info: mode: chat - # alias: summarizer-large -> gen / qwen3.5-122-a10b (operator 2026-06-19). For heavier - # summarization that wants the 122B Qwopus instead of granite-8b. Thinking OFF (matches - # gen). Keep api_base (:8013) + enable_thinking in sync with the gen record below. + # alias: summarizer-large -> gen / qwen3.6-27b-aeon (operator 2026-07-05). For heavier + # summarization that wants the AEON 27B `gen` model instead of granite-8b. Thinking OFF + # (matches gen). Keep api_base (:8015) + enable_thinking in sync with the gen record below. - model_name: summarizer-large litellm_params: - model: hosted_vllm/qwen3.5-122-a10b - api_base: http://10.250.50.54:8013/v1 + model: hosted_vllm/qwen3.6-27b-aeon + api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.0 temperature: 0.7 @@ -104,82 +104,24 @@ model_list: model_info: mode: chat - # --- Qwopus3.5-122B-A10B (Kimi-distilled, abliterated, NVFP4, VISION-INTACT) — the - # general / `gen` model on ana-ml2 GPU 0. Replaced the bjk110 text-only qwen3.5-122b - # 2026-06-19 (which had replaced mistral-small-4). Served on :8013 via vLLM as plain - # multimodal (no text-only patch), served-name qwen3.5-122-a10b — so these records - # route UNCHANGED. Full 256K (262144) @ fp8 KV + CUDA graphs (92.7 tok/s warm); - # tool-calling via qwen3_coder. Thinking split = chat_template_kwargs.enable_thinking - # + --reasoning-parser qwen3. One upstream fanned out under qwen3.5-122-a10b[-reasoning] - # + aliases qwen-large[-reasoning] + gen[-reasoning]; -reasoning variants enable - # thinking. Keep api_base in sync. - # presence_penalty: 1.0 on ALL these qwen3.5-122-a10b records (+ summarizer-large - # above) — anti-repetition-loop damper for the abliterated/NVFP4 tendency (operator - # 2026-06-27). Gateway-tunable default (callers can override); bake the validated - # value into the vLLM serving def (stacks/qwen3.5-122b, --override-generation-config) - # once confirmed, to also cover direct (non-gateway) callers. - # ⚠️ Worldtree CHARACTER backend (was bound to mistral-small-4) is dark until - # repointed — operator-acknowledged. --- - - model_name: qwen3.5-122-a10b - litellm_params: - model: hosted_vllm/qwen3.5-122-a10b - api_base: http://10.250.50.54:8013/v1 - api_key: os.environ/VLLM_API_KEY - presence_penalty: 1.0 - temperature: 0.7 - top_p: 0.8 - extra_body: - top_k: 20 - chat_template_kwargs: - enable_thinking: false - model_info: - mode: chat - - model_name: qwen3.5-122-a10b-reasoning - litellm_params: - model: hosted_vllm/qwen3.5-122-a10b - api_base: http://10.250.50.54:8013/v1 - api_key: os.environ/VLLM_API_KEY - presence_penalty: 1.0 - temperature: 0.6 - top_p: 0.95 - extra_body: - top_k: 20 - chat_template_kwargs: - enable_thinking: true - model_info: - mode: chat - - model_name: qwen-large - litellm_params: - model: hosted_vllm/qwen3.5-122-a10b - api_base: http://10.250.50.54:8013/v1 - api_key: os.environ/VLLM_API_KEY - presence_penalty: 1.0 - temperature: 0.7 - top_p: 0.8 - extra_body: - top_k: 20 - chat_template_kwargs: - enable_thinking: false - model_info: - mode: chat - - model_name: qwen-large-reasoning - litellm_params: - model: hosted_vllm/qwen3.5-122-a10b - api_base: http://10.250.50.54:8013/v1 - api_key: os.environ/VLLM_API_KEY - presence_penalty: 1.0 - temperature: 0.6 - top_p: 0.95 - extra_body: - top_k: 20 - chat_template_kwargs: - enable_thinking: true - model_info: - mode: chat + # --- Qwen3.6-27B AEON (uncensored/abliterated, NVFP4 ModelOpt, VISION-INTACT image+video) + # — the general / `gen` model on ana-ml2 GPU 0. REPLACED qwopus3.5-122b 2026-07-05 + # (operator: displace qwopus, this assumes gen/gen-reasoning). Dense 27B, qwen3_5 + # GDN-hybrid (qwopus's little sibling), native MTP head. Served on :8015 via vLLM, + # served-name qwen3.6-27b-aeon, MTP OFF (spec-decode hurts concurrent aggregate — the + # MTP twin is char-rp below). Thinking split = chat_template_kwargs.enable_thinking + + # --reasoning-parser qwen3; tool-calling qwen3_coder. gen / gen-reasoning + summarizer- + # large route here; -reasoning enables thinking. Keep api_base (:8015) in sync. + # RETIRED with the displacement (→ 404, callers migrate to gen): qwen3.5-122-a10b + # [-reasoning] + qwen-large[-reasoning] — they named a 122B that no longer exists; + # aliasing a 27B under those is the naming footgun the qwen36-vl stack warns against. + # presence_penalty: 1.0 INHERITED from qwopus (same-family abliterated/NVFP4 anti- + # repetition damper, operator 2026-06-27) — RE-VALIDATE for AEON; NOT yet confirmed + # for this model's repetition behavior. --- - model_name: gen litellm_params: - model: hosted_vllm/qwen3.5-122-a10b - api_base: http://10.250.50.54:8013/v1 + model: hosted_vllm/qwen3.6-27b-aeon + api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.0 temperature: 0.7 @@ -192,8 +134,10 @@ model_list: mode: chat - model_name: gen-reasoning litellm_params: - model: hosted_vllm/qwen3.5-122-a10b - api_base: http://10.250.50.54:8013/v1 + # Distinct served-name so a thinking-off `gen` request can't mutate this deployment's + # enable_thinking (shared-config-mutation footgun). Same backend :8015, different model id. + model: hosted_vllm/qwen3.6-27b-aeon-thinking + api_base: http://10.250.50.54:8015/v1 api_key: os.environ/VLLM_API_KEY presence_penalty: 1.0 temperature: 0.6 @@ -204,6 +148,51 @@ model_list: enable_thinking: true model_info: mode: chat + # char-rp -> the native-MTP single-seat RP twin (:8016, served qwen3.6-27b-aeon-rp). + # SAME weights as gen, MTP ON (qwen3_5_mtp n=3) for single-stream RP latency. RP-shaped + # sampler defaults (Worldtree character-rp role / callers override); thinking OFF. + # ⚠️ MTP silently drops min_p/logit_bias — don't rely on those through char-rp. + - model_name: char-rp + litellm_params: + model: hosted_vllm/qwen3.6-27b-aeon-rp + api_base: http://10.250.50.54:8016/v1 + api_key: os.environ/VLLM_API_KEY + # Qwen3.x non-thinking RP recs (operator 2026-07-05, adapted from Qwen + community + # RP testing). presence_penalty light (0.1) to reduce topic drift; repetition_penalty + # 1.05. min_p skipped (Qwen rec + MTP drops it anyway). DRY off (Qwen3.x artifacts). + temperature: 0.7 + top_p: 0.8 + presence_penalty: 0.1 + extra_body: + top_k: 20 + repetition_penalty: 1.05 + chat_template_kwargs: + enable_thinking: false + model_info: + mode: chat + # char-rp-reasoning -> same RP seat (:8016, MTP), thinking ON. Reasoning-profile + # sampler (lower temp than char-rp for coherent thought); tunable. ⚠️ The thinking + # TRACE does not yet surface in reasoning_content (chat_template injects in the + # prompt → parser drops the span); pending a template fix, NOT a parser swap. + - model_name: char-rp-reasoning + litellm_params: + # Distinct served-name so a thinking-off char-rp request can't clobber this to + # enable_thinking:false (the bug that broke it). Same backend :8016, different model id. + model: hosted_vllm/qwen3.6-27b-aeon-rp-thinking + api_base: http://10.250.50.54:8016/v1 + api_key: os.environ/VLLM_API_KEY + # Same RP profile as char-rp but thinking-mode top_p 0.95 (Qwen thinking rec) + focused + # temp 0.6. (Reasoning-trace surfacing still pending the parser/template fix.) + temperature: 0.6 + top_p: 0.95 + presence_penalty: 0.1 + extra_body: + top_k: 20 + repetition_penalty: 1.05 + chat_template_kwargs: + enable_thinking: true + model_info: + mode: chat # --- Selene 1 Mini 8B (AtlaAI judge, FP8) — restored on GPU1 after the # llama-swap teardown (was the Q6_K GGUF in the swap zoo). vLLM dynamic fp8, diff --git a/stacks/qwen36-27b-aeon/.env.example b/stacks/qwen36-27b-aeon/.env.example new file mode 100644 index 0000000..a20b896 --- /dev/null +++ b/stacks/qwen36-27b-aeon/.env.example @@ -0,0 +1,49 @@ +# qwen36-27b-aeon tunables — copy to .env on the host, edit there (never commit .env). +# The real .env lives at /opt/docker/compose/qwen36-27b-aeon/.env on ana-ml2. + +# Per-service vLLM image. Both on latest (0.24.0) after validating it on rp. Reasoning works +# on 0.24.0 (and did on 0.23.0 too — the trace bug was a LiteLLM shared-config mutation, not +# vLLM). Rollback: set either back to vllm/vllm-openai:known-good-0.23.0 (tagged on the host). +AEON_GEN_IMAGE=vllm/vllm-openai:latest +AEON_RP_IMAGE=vllm/vllm-openai:latest + +# Shared vLLM API key (matches the litellm VLLM_API_KEY). Blank = no auth. +API_KEY= + +# Both instances pin to the same physical GPU (the freed qwopus slot). +AEON_GPU_ID=0 + +# ── General serve (MTP off) → gen / gen-reasoning / summarizer-large ── +AEON_GEN_CONTAINER_NAME=vllm-aeon-gen +AEON_GEN_PORT=8015 +AEON_GEN_SERVED_NAME=qwen3.6-27b-aeon +AEON_GEN_SERVED_NAME_THINK=qwen3.6-27b-aeon-thinking +AEON_GEN_MODEL=/tank/aimodels/qwen36-27b-aeon-nvfp4 +AEON_GEN_GPU_MEM_UTIL=0.45 +AEON_GEN_MAX_MODEL_LEN=131072 +AEON_GEN_MAX_NUM_SEQS=16 +AEON_GEN_KV_CACHE_DTYPE=fp8 +# Reasoning parser — KEEP qwen3. This checkpoint's chat_template.jinja injects the opening +# into the PROMPT (output has only ). qwen3 handles the non-thinking path +# correctly. NOTE: enable_thinking:true does NOT surface reasoning_content under EITHER qwen3 +# or deepseek_r1 (the reasoning is generated but dropped) — that needs a chat_template fix, +# NOT a parser swap. deepseek_r1 was tested and is WORSE: it breaks the non-thinking path +# too (both content and reasoning_content come back empty). Do not use deepseek_r1 here. +AEON_GEN_REASONING_PARSER=qwen3 + +# ── RP seat (native MTP on) → char-rp ── +AEON_RP_CONTAINER_NAME=vllm-aeon-rp +AEON_RP_PORT=8016 +AEON_RP_SERVED_NAME=qwen3.6-27b-aeon-rp +AEON_RP_SERVED_NAME_THINK=qwen3.6-27b-aeon-rp-thinking +# Full 27GB variant by default; switch to the 21GB XS for ~6GB more co-location margin: +# AEON_RP_MODEL=/tank/aimodels/qwen36-27b-aeon-nvfp4-xs +AEON_RP_MODEL=/tank/aimodels/qwen36-27b-aeon-nvfp4 +AEON_RP_GPU_MEM_UTIL=0.40 +AEON_RP_MAX_MODEL_LEN=65536 +AEON_RP_MAX_NUM_SEQS=2 +AEON_RP_KV_CACHE_DTYPE=fp8 +AEON_RP_REASONING_PARSER=qwen3 +# Native MTP head. If stock vLLM names it differently, try method=mtp. +AEON_RP_SPEC_METHOD=qwen3_5_mtp +AEON_RP_SPEC_TOKENS=3 diff --git a/stacks/qwen36-27b-aeon/compose.yaml b/stacks/qwen36-27b-aeon/compose.yaml new file mode 100644 index 0000000..3b77301 --- /dev/null +++ b/stacks/qwen36-27b-aeon/compose.yaml @@ -0,0 +1,214 @@ +# qwen36-27b-aeon — AEON-7/Qwen3.6-27B-AEON-Ultimate-Uncensored on ana-ml2 GPU 0, +# REPLACING qwopus3.5-122b as the `gen` model (operator 2026-07-05). +# +# Dense 27B, qwen3_5 GDN-hybrid arch (full-attn + Gated DeltaNet SSM) — same family +# as qwopus, VISION-INTACT (Qwen3_5ForConditionalGeneration, vision tower preserved +# at bf16), abliterated (abliterix v1.4, 0/100 refusals), native MTP head grafted, +# Apache-2.0, 131K default ctx. Served NVFP4 (ModelOpt) on Blackwell's FP4 cores. +# +# TWO CO-LOCATED INSTANCES on GPU 0 (operator wants both behaviours resident at once; +# MTP is a serve-time config, NOT per-request, so one endpoint can't do both): +# vllm-aeon-gen (:8015, served qwen3.6-27b-aeon) — MTP OFF, general/concurrent +# serve. Backs gateway gen / gen-reasoning / summarizer-large. +# vllm-aeon-rp (:8016, served qwen3.6-27b-aeon-rp) — native MTP ON (qwen3_5_mtp +# n=3), low-concurrency single-seat RP. Backs gateway char-rp. +# Why MTP off for the general serve: measured on qwopus, MTP helps single-stream +# (+12% N=1) but HURTS moderate concurrency (-15..-20% N=4) and silently drops +# min_p/logit_bias — wrong for a shared multi-consumer endpoint. Right only for a +# dedicated single-stream seat (the RP one). [[reference_gen_qwopus_122b]] +# +# VRAM budget (2 weight copies, no sharing): full NVFP4 = 27GB ea. gen util 0.45 +# (~43GB) + rp util 0.40 (~38GB) = ~81GB / 96GB, ~15GB margin. depends_on: +# service_healthy sequences gen-first so the util reservation doesn't race → OOM. +# If margin bites at warmup (vision-encoder + big-vocab sampler warmup, cf. qwen36-vl), +# point AEON_RP_MODEL at the 21GB XS variant (frees ~6GB) via .env — no compose edit. +# +# NVFP4 is ModelOpt format → --quantization modelopt (vLLM also auto-detects; explicit +# is belt-and-suspenders). --mamba-cache-dtype float32 for the GDN/SSM state (AEON +# deploy guide + vLLM recipe Mamba-cache note). Tool-calling qwen3_coder + reasoning +# qwen3 (per the AEON card), same as qwopus. +# +# ⚠️ DEPLOYABILITY — load-test before trusting: multimodal + ModelOpt-NVFP4 on THIS +# brand-new arch, and native qwen3_5_mtp spec-decode, are unproven on our stock vLLM +# image. If stock can't serve it, the AEON patched image (ghcr.io/aeon-7/aeon-vllm- +# ultimate, PRs #41703/#40898) is the fallback — but that's really for DFlash; native +# MTP + base inference should ride stock >= 0.23.0. Set AEON_IMAGE in .env. +# +# REVERT: `docker compose down` here + `docker compose up -d` the qwopus3.5-122b stack +# (still staged) + revert the litellm gen/gen-reasoning/summarizer-large records. +# All tunables in .env — edit that, not this file. + +name: qwen36-27b-aeon + +services: + # ── General serve — MTP OFF, concurrent. gen / gen-reasoning / summarizer-large. ── + vllm-aeon-gen: + # Per-service image so gen can stay pinned to a known-good vLLM while rp tests a new one. + image: ${AEON_GEN_IMAGE:-vllm/vllm-openai:latest} + container_name: ${AEON_GEN_CONTAINER_NAME:-vllm-aeon-gen} + restart: unless-stopped + ipc: host + ports: + - "${AEON_GEN_PORT:-8015}:8000" + volumes: + - /tank/aimodels/huggingface:/hfcache + - ${AEON_GEN_MODEL:-/tank/aimodels/qwen36-27b-aeon-nvfp4}:/model:ro + environment: + - HF_HOME=/hfcache + - HF_HUB_CACHE=/hfcache/hub + - VLLM_API_KEY=${API_KEY:-} + # Reclaims PyTorch reserved-but-unallocated fragmentation so the co-located + # util split doesn't strand VRAM (same knob qwopus needed for the MoE workspace). + - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + command: + - /model + # TWO served-names: base + a `-thinking` alias. LiteLLM keys deployments by + # (model, api_base), so gen and gen-reasoning MUST use distinct model names or a + # thinking-off request mutates the shared litellm_params and clobbers the other's + # enable_thinking (the shared-config-mutation footgun). gen-reasoning routes to the + # `-thinking` name; gen/summarizer-large route to the base name. + - --served-model-name + - ${AEON_GEN_SERVED_NAME:-qwen3.6-27b-aeon} + - ${AEON_GEN_SERVED_NAME_THINK:-qwen3.6-27b-aeon-thinking} + - --host + - 0.0.0.0 + - --port + - "8000" + - --quantization + - modelopt + - --gpu-memory-utilization + - ${AEON_GEN_GPU_MEM_UTIL:-0.45} + - --max-model-len + - ${AEON_GEN_MAX_MODEL_LEN:-131072} + # Keep concurrency modest: big-vocab sampler warmup allocates a large tensor + # (qwen36-vl OOM'd at the default 1024 on a shared GPU). 16 is ample here. + - --max-num-seqs + - ${AEON_GEN_MAX_NUM_SEQS:-16} + - --max-num-batched-tokens + - "16384" + - --trust-remote-code + - --dtype + - auto + # GDN/SSM (Gated DeltaNet) state cache — float32 per the AEON deploy guide. + - --mamba-cache-dtype + - float32 + - --kv-cache-dtype + - ${AEON_GEN_KV_CACHE_DTYPE:-fp8} + - --enable-prefix-caching + - --enable-chunked-prefill + - --limit-mm-per-prompt + - '{"image": 4}' + - --reasoning-parser + - ${AEON_GEN_REASONING_PARSER:-qwen3} + - --enable-auto-tool-choice + - --tool-call-parser + - qwen3_coder + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${AEON_GPU_ID:-0}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 900s + networks: + - tnet + labels: + - homepage.group=AI Systems + - homepage.name=Qwen3.6-27B AEON (NVFP4, vision) — gen + - homepage.icon=mdi-creation + - homepage.description=Uncensored Qwen3.6-27B multimodal NVFP4, the `gen` model (ana-ml2 GPU 0) + - homepage.href=http://10.250.50.54:${AEON_GEN_PORT:-8015}/docs + + # ── RP seat — native MTP ON, low concurrency, single-seat. char-rp. ── + vllm-aeon-rp: + image: ${AEON_RP_IMAGE:-vllm/vllm-openai:latest} + container_name: ${AEON_RP_CONTAINER_NAME:-vllm-aeon-rp} + restart: unless-stopped + ipc: host + # Sequence AFTER the general serve is healthy so the two util reservations on the + # shared GPU don't race into an OOM (gen reserves its 0.45 first, then rp its 0.40). + depends_on: + vllm-aeon-gen: + condition: service_healthy + ports: + - "${AEON_RP_PORT:-8016}:8000" + volumes: + - /tank/aimodels/huggingface:/hfcache + # Point at the XS (21GB) variant via .env to buy ~6GB co-location margin. + - ${AEON_RP_MODEL:-/tank/aimodels/qwen36-27b-aeon-nvfp4}:/model:ro + environment: + - HF_HOME=/hfcache + - HF_HUB_CACHE=/hfcache/hub + - VLLM_API_KEY=${API_KEY:-} + - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + command: + - /model + # base + `-thinking` alias (see the gen note): char-rp -> base, char-rp-reasoning -> -thinking. + - --served-model-name + - ${AEON_RP_SERVED_NAME:-qwen3.6-27b-aeon-rp} + - ${AEON_RP_SERVED_NAME_THINK:-qwen3.6-27b-aeon-rp-thinking} + - --host + - 0.0.0.0 + - --port + - "8000" + - --quantization + - modelopt + - --gpu-memory-utilization + - ${AEON_RP_GPU_MEM_UTIL:-0.40} + - --max-model-len + - ${AEON_RP_MAX_MODEL_LEN:-65536} + # Single-seat: low concurrency keeps warmup + KV small so it fits alongside gen. + - --max-num-seqs + - ${AEON_RP_MAX_NUM_SEQS:-2} + - --trust-remote-code + - --dtype + - auto + - --mamba-cache-dtype + - float32 + - --kv-cache-dtype + - ${AEON_RP_KV_CACHE_DTYPE:-fp8} + - --enable-prefix-caching + - --limit-mm-per-prompt + - '{"image": 4}' + - --reasoning-parser + - ${AEON_RP_REASONING_PARSER:-qwen3} + # Native MTP speculative decode (the grafted head). n=3 per the AEON card's + # measured accept length (~3.3/3). MTP is why this seat exists separately. + - --speculative-config + - '{"method": "${AEON_RP_SPEC_METHOD:-qwen3_5_mtp}", "num_speculative_tokens": ${AEON_RP_SPEC_TOKENS:-3}}' + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${AEON_GPU_ID:-0}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 900s + networks: + - tnet + labels: + - homepage.group=AI Systems + - homepage.name=Qwen3.6-27B AEON RP (NVFP4 + MTP) — char-rp + - homepage.icon=mdi-drama-masks + - homepage.description=Uncensored Qwen3.6-27B, native MTP single-seat RP (char-rp), ana-ml2 GPU 0 + - homepage.href=http://10.250.50.54:${AEON_RP_PORT:-8016}/docs + +networks: + tnet: + name: traefik-net + external: true