diff --git a/stacks/qwopus3.5-122b/.env.example b/stacks/qwopus3.5-122b/.env.example new file mode 100644 index 0000000..a93b6c2 --- /dev/null +++ b/stacks/qwopus3.5-122b/.env.example @@ -0,0 +1,28 @@ +# qwopus3.5-122b (OpenYourMind Qwopus3.5-122B-A10B Kimi-distilled abliterated NVFP4, +# vision-intact) — ana-ml2 GPU 0 tunables. Real .env at /opt/docker/compose/qwopus3.5-122b/.env + +# STABLE image + fp8 KV reaches full 256K: the KV pool already held ~222k tokens, so fp8 +# (near-lossless, half the bytes/token) clears 262144 with ~2x concurrency. Nightly + +# turboquant_4bit_nc would buy ~5x concurrency at 256K but adds a 4-bit recall risk + FA2 +# fallback + nightly instability — not needed for 256K itself. +QWOPUS_IMAGE=vllm/vllm-openai:latest +QWOPUS_CONTAINER_NAME=vllm-qwopus35-122b +QWOPUS_KV_CACHE_DTYPE=fp8 + +# Reuse :8013 (the bjk110 qwen3.5-122b port, now retired) so the litellm records route +# here unchanged. Served under qwen3.5-122-a10b (the operator's gen records). +QWOPUS_PORT=8013 +QWOPUS_SERVED_NAME=qwen3.5-122-a10b +QWOPUS_GPU_ID=0 + +# Vision-intact NVFP4 (≈82GB incl. bf16 vision tower) on the 96GB Blackwell. CUDA graphs +# ON (no --enforce-eager) for decode throughput. util 0.95 — 0.96 OOM'd by 0.1GB on the +# 3.09GB FusedMoE transient workspace (the hard floor; the card can't reach 0 free), so +# expandable_segments (compose env) reclaims PyTorch fragmentation + 0.95 adds margin. +# max-num-seqs 16 lets short requests fan out (~16x32k); 256K requests pool-limit to ~3.5x. +QWOPUS_GPU_MEM_UTIL=0.95 +QWOPUS_MAX_MODEL_LEN=262144 +QWOPUS_MAX_NUM_SEQS=16 + +# Optional upstream vLLM API key (empty = no auth; internal net only). +API_KEY= diff --git a/stacks/qwopus3.5-122b/compose.yaml b/stacks/qwopus3.5-122b/compose.yaml new file mode 100644 index 0000000..e8a03ab --- /dev/null +++ b/stacks/qwopus3.5-122b/compose.yaml @@ -0,0 +1,108 @@ +# qwopus3.5-122b — OpenYourMind/Qwopus3.5-122B-A10B-Kimi-K2.6-destilled-abliterated-NVFP4 +# on ana-ml2 GPU 0, REPLACING the bjk110 text-only qwen3.5-122b as the `gen` model +# (operator 2026-06-19: "already ablated, already quanted, vision tower intact"). +# +# Qwen3.5-122B-A10B MoE, Kimi-K2.6-distilled + abliterated, NVFP4 — and crucially +# VISION-INTACT (Qwen3_5MoeForConditionalGeneration + vision_config). So it serves as +# plain MULTIMODAL (no text-only patch, unlike the bjk110 checkpoint which had its +# vision weights stripped). vLLM carries the arch natively. +# +# Served under --served-model-name qwen3.5-122-a10b so the existing litellm records +# (gen / gen-reasoning / qwen3.5-122-a10b[-reasoning] / qwen-large[-reasoning]) route +# here UNCHANGED — the operator's "replace those records with this model". The thinking +# split (chat_template_kwargs.enable_thinking) + tool-calling (qwen3_coder — the +# OpenYourMind card's specified parser for this checkpoint's XML tool calls). +# +# REVERT: down this; the bjk110 qwen3.5-122b stack is still staged. +# Tunables in .env. + +name: qwopus3.5-122b + +services: + vllm-qwopus35-122b: + image: ${QWOPUS_IMAGE:-vllm/vllm-openai:latest} + container_name: ${QWOPUS_CONTAINER_NAME:-vllm-qwopus35-122b} + restart: unless-stopped + ipc: host + ports: + - "${QWOPUS_PORT:-8013}:8000" + volumes: + - /tank/aimodels/huggingface:/hfcache + - /tank/aimodels/qwopus3.5-122b-nvfp4:/model:ro + environment: + - HF_HOME=/hfcache + - HF_HUB_CACHE=/hfcache/hub + - VLLM_API_KEY=${API_KEY:-} + # Reclaims PyTorch's reserved-but-unallocated fragmentation (4.2GB was stranded at + # util 0.96, starving the FusedMoE workspace → OOM by 0.1GB). Lets the 3.09GB MoE + # workspace allocate cleanly. Same knob the bjk110 qwen3.5-122b stack ran. + - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + command: + - /model + - --served-model-name + - ${QWOPUS_SERVED_NAME:-qwen3.5-122-a10b} + - --host + - 0.0.0.0 + - --port + - "8000" + - --gpu-memory-utilization + - ${QWOPUS_GPU_MEM_UTIL:-0.92} + - --max-model-len + - ${QWOPUS_MAX_MODEL_LEN:-131072} + - --max-num-seqs + - ${QWOPUS_MAX_NUM_SEQS:-8} + - --max-num-batched-tokens + - "32768" + - --trust-remote-code + - --dtype + - auto + - --enable-prefix-caching + - --enable-chunked-prefill + # FULL 256K context on the STABLE image. fp8 KV (near-lossless) measured an 11.8GB + # pool = 934,600 tokens = 3.5x concurrency at the full 262144 window. CUDA graphs ON + # (no --enforce-eager) for decode tok/s. BINDING LIMIT = the FusedMoE transient + # workspace (3.09GB, allocated OUTSIDE vLLM's budget into free VRAM): at util 0.96 + # only 2.99GB was free → OOM by 0.1GB, worsened by 4.2GB PyTorch fragmentation. + # FIX = expandable_segments (env above, reclaims the fragmentation) + util 0.95 for + # margin. The card can't go to 0 free — this workspace is the floor. video kept + # ENABLED (operator wants it; banked at util 0.95 with headroom) — the video encoder + # profiling eats into the budget so KV concurrency drops some, but stays well above 2x. + - --kv-cache-dtype + - ${QWOPUS_KV_CACHE_DTYPE:-fp8} + - --limit-mm-per-prompt + - '{"image": 2, "video": 1}' + # reasoning split + tool-calling. The OpenYourMind card specifies qwen3_coder + # as the tool-call parser for this checkpoint (Qwen3.5 XML tool-call format). + - --reasoning-parser + - qwen3 + - --enable-auto-tool-choice + - --tool-call-parser + - qwen3_coder + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${QWOPUS_GPU_ID:-0}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 900s + networks: + - tnet + labels: + - homepage.group=AI Systems + - homepage.name=Qwopus3.5-122B-A10B (abliterated NVFP4, vision) + - homepage.icon=mdi-creation + - homepage.description=Kimi-distilled abliterated Qwen3.5-122B-A10B NVFP4, vision-intact, the `gen` model (ana-ml2 GPU 0) + - homepage.href=http://10.250.50.54:${QWOPUS_PORT:-8013}/docs + +networks: + tnet: + name: traefik-net + external: true