vh
89c83c4271
bjk110/Qwen3.5-122B-A10B-abliterated-NVFP4 on ana-ml2 GPU 0 (heretic downed): - stacks/qwen3.5-122b/ — vLLM serve via the repo's text-only patch (Qwen3.5 MoE is a multimodal arch but this checkpoint is text-only weights), --reasoning-parser qwen3, GPU 0 pin, :8013; entrypoint+patch mounted from the model dir. - serve-qwen3.5-122b.yaml — displace heretic + serve + verify. - litellm: REMOVED dead mistral-small-4 / -reasoning; added qwen3.5-122-a10b[-reasoning] + aliases qwen-large[-reasoning] + repointed gen[-reasoning] -> qwen (thinking split via chat_template_kwargs.enable_thinking + --reasoning-parser qwen3). Verified live: qwen healthy on :8013; gen / qwen-large / qwen3.5-122-a10b route, and gen-reasoning returns reasoning_content; mistral-small-4 removed. NOTE: Worldtree character backend (was bound to mistral-small-4) is dark until repointed (operator-acknowledged).
82 lines
3.3 KiB
YAML
82 lines
3.3 KiB
YAML
# qwen3.5-122b — bjk110/Qwen3.5-122B-A10B-abliterated-NVFP4 on ana-ml2 GPU 0,
|
|
# REPLACING mistral-small-4 (operator 2026-06-19: down the heretic, serve this as
|
|
# the new general/`gen` model). Abliterated Qwen3.5 MoE (256 experts, 10B active),
|
|
# NVFP4 (compressed-tensors), HF format.
|
|
#
|
|
# SERVING (per the repo's serving/): Qwen3.5 MoE is a MULTIMODAL arch but this
|
|
# checkpoint is text-only, so vLLM needs the repo's text-only PATCH applied before
|
|
# startup. We reuse the repo's entrypoint.sh (applies the patch, then runs vLLM) and
|
|
# vllm_patches/, mounted from the downloaded model dir — keeps patch+checkpoint
|
|
# version-coupled. Thinking split via litellm extra_body chat_template_kwargs
|
|
# (enable_thinking) + --reasoning-parser qwen3 (mirrors the qwen3.6-35b-a3b pattern).
|
|
#
|
|
# DISPLACEMENT: GPU 0 fits one mistral-class model; bring this up only after downing
|
|
# mistral-small-4-heretic. REVERT = down this, `up -d` the heretic stack.
|
|
#
|
|
# NOTE: image is vllm/vllm-openai:latest per the repo (the patch targets latest) —
|
|
# MUTABLE tag; pin a digest once a known-good version is established.
|
|
#
|
|
# Tunables in .env.
|
|
|
|
name: qwen3.5-122b
|
|
|
|
services:
|
|
vllm-qwen35-122b:
|
|
image: ${QWEN35_IMAGE:-vllm/vllm-openai:latest}
|
|
container_name: ${QWEN35_CONTAINER_NAME:-vllm-qwen35-122b}
|
|
restart: unless-stopped
|
|
ipc: host
|
|
ports:
|
|
- "${QWEN35_PORT:-8013}:8000"
|
|
environment:
|
|
- ROLE=head
|
|
- TP_SIZE=1
|
|
- MODEL_CONTAINER_PATH=/models/qwen
|
|
- SERVED_MODEL_NAME=${QWEN35_SERVED_NAME:-qwen3.5-122-a10b}
|
|
- HOST_PORT=8000
|
|
- MAX_MODEL_LEN=${QWEN35_MAX_MODEL_LEN:-131072}
|
|
- MAX_NUM_SEQS=${QWEN35_MAX_NUM_SEQS:-4}
|
|
- GPU_MEMORY_UTILIZATION=${QWEN35_GPU_MEM_UTIL:-0.90}
|
|
- MAX_NUM_BATCHED_TOKENS=${QWEN35_MAX_NUM_BATCHED_TOKENS:-32768}
|
|
# --reasoning-parser qwen3 surfaces <think>…</think> as reasoning_content;
|
|
# the thinking on/off itself is per-request (litellm chat_template_kwargs).
|
|
- VLLM_EXTRA_ARGS=--reasoning-parser qwen3 --enable-chunked-prefill
|
|
- NVIDIA_VISIBLE_DEVICES=0
|
|
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
|
- VLLM_API_KEY=${API_KEY:-}
|
|
volumes:
|
|
# checkpoint + the repo's entrypoint/patch (downloaded with the model).
|
|
- /tank/aimodels/qwen3.5-122b-a10b-nvfp4:/models/qwen:ro
|
|
- /tank/aimodels/qwen3.5-122b-a10b-nvfp4/serving/entrypoint.sh:/entrypoint.sh:ro
|
|
- /tank/aimodels/qwen3.5-122b-a10b-nvfp4/vllm_patches:/patches:ro
|
|
- /tank/aimodels/qwen3.5-122b-a10b-nvfp4/.cache/vllm:/root/.cache/vllm
|
|
entrypoint: ["bash", "/entrypoint.sh"]
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
device_ids:
|
|
- "${QWEN35_GPU_ID:-0}"
|
|
capabilities:
|
|
- gpu
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 900s
|
|
networks:
|
|
- tnet
|
|
labels:
|
|
- homepage.group=AI Systems
|
|
- homepage.name=Qwen3.5-122B-A10B (abliterated NVFP4)
|
|
- homepage.icon=mdi-creation
|
|
- homepage.description=Abliterated Qwen3.5 122B-A10B NVFP4, the new `gen` model (ana-ml2 GPU 0)
|
|
- homepage.href=http://10.250.50.54:${QWEN35_PORT:-8013}/docs
|
|
|
|
networks:
|
|
tnet:
|
|
name: traefik-net
|
|
external: true
|