feat(qwen3.5-122b): replace mistral-small-4 as gen (abliterated NVFP4, text-only)
bjk110/Qwen3.5-122B-A10B-abliterated-NVFP4 on ana-ml2 GPU 0 (heretic downed): - stacks/qwen3.5-122b/ — vLLM serve via the repo's text-only patch (Qwen3.5 MoE is a multimodal arch but this checkpoint is text-only weights), --reasoning-parser qwen3, GPU 0 pin, :8013; entrypoint+patch mounted from the model dir. - serve-qwen3.5-122b.yaml — displace heretic + serve + verify. - litellm: REMOVED dead mistral-small-4 / -reasoning; added qwen3.5-122-a10b[-reasoning] + aliases qwen-large[-reasoning] + repointed gen[-reasoning] -> qwen (thinking split via chat_template_kwargs.enable_thinking + --reasoning-parser qwen3). Verified live: qwen healthy on :8013; gen / qwen-large / qwen3.5-122-a10b route, and gen-reasoning returns reasoning_content; mistral-small-4 removed. NOTE: Worldtree character backend (was bound to mistral-small-4) is dark until repointed (operator-acknowledged).
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
# Displace mistral-small-4 (heretic) on ana-ml2 GPU 0 and serve
|
||||
# bjk110/Qwen3.5-122B-A10B-abliterated-NVFP4 (text-only) as the new `gen` model
|
||||
# (operator 2026-06-19). Weights pre-staged at /tank/aimodels/qwen3.5-122b-a10b-nvfp4
|
||||
# (incl. the repo's serving/entrypoint.sh + vllm_patches/ that the compose mounts).
|
||||
#
|
||||
# ⚠️ Downing mistral-small-4 takes down the Worldtree CHARACTER backend (vision-intact)
|
||||
# until it's repointed — operator-acknowledged. REVERT = down qwen, up -d the heretic.
|
||||
#
|
||||
# scripts/elway ana-ml2 --playbook playbooks/serve-qwen3.5-122b.yaml
|
||||
|
||||
vars:
|
||||
compose_dir: /opt/docker/compose/qwen3.5-122b
|
||||
heretic_dir: /opt/docker/compose/mistral-small-4-heretic
|
||||
model_dir: /tank/aimodels/qwen3.5-122b-a10b-nvfp4
|
||||
host_port: "8013"
|
||||
|
||||
steps:
|
||||
- name: Verify NVFP4 weights + the repo's patch/entrypoint are staged
|
||||
shell: |
|
||||
test -f {{ model_dir }}/model.safetensors.index.json \
|
||||
&& test -f {{ model_dir }}/serving/entrypoint.sh \
|
||||
&& test -f {{ model_dir }}/vllm_patches/patch_qwen35_moe_text.py
|
||||
changed_when: "false"
|
||||
|
||||
- name: Ensure vLLM compile-cache dir exists (writable)
|
||||
shell: mkdir -p {{ model_dir }}/.cache/vllm
|
||||
creates: "{{ model_dir }}/.cache/vllm"
|
||||
|
||||
- name: Ensure compose dir exists
|
||||
shell: mkdir -p {{ compose_dir }}
|
||||
creates: "{{ compose_dir }}"
|
||||
|
||||
- name: Upload compose.yaml
|
||||
upload:
|
||||
src: stacks/qwen3.5-122b/compose.yaml
|
||||
dest: "{{ compose_dir }}/compose.yaml"
|
||||
mode: "0644"
|
||||
|
||||
- name: Seed .env from template (only if absent)
|
||||
upload:
|
||||
src: stacks/qwen3.5-122b/.env.example
|
||||
dest: "{{ compose_dir }}/.env"
|
||||
mode: "0644"
|
||||
when: "[ ! -f {{ compose_dir }}/.env ]"
|
||||
|
||||
- name: Displace — down mistral-small-4-heretic (frees GPU 0; no-op if down)
|
||||
shell: cd {{ heretic_dir }} && docker compose down
|
||||
|
||||
- name: Bring up qwen3.5-122b
|
||||
shell: cd {{ compose_dir }} && docker compose up -d
|
||||
|
||||
- name: Wait for vLLM /health (allow ~15 min for patch + NVFP4 MoE load + warmup)
|
||||
shell: |
|
||||
for i in $(seq 1 180); do
|
||||
curl -sf -o /dev/null --max-time 3 http://localhost:{{ host_port }}/health && exit 0
|
||||
sleep 5
|
||||
done
|
||||
exit 1
|
||||
changed_when: "false"
|
||||
|
||||
verify:
|
||||
- name: /health returns 200
|
||||
shell: curl -sf -o /dev/null http://localhost:{{ host_port }}/health
|
||||
changed_when: "false"
|
||||
|
||||
- name: served model id is qwen3.5-122-a10b
|
||||
shell: curl -sf http://localhost:{{ host_port }}/v1/models | grep -q qwen3.5-122-a10b
|
||||
changed_when: "false"
|
||||
|
||||
- name: container running
|
||||
shell: docker inspect vllm-qwen35-122b --format '{{.State.Status}}' | grep -q running
|
||||
changed_when: "false"
|
||||
@@ -78,51 +78,73 @@ model_list:
|
||||
model_info:
|
||||
mode: chat
|
||||
|
||||
# --- Mistral Small 4 (official NVFP4) — creative-writing / general text. 119B
|
||||
# MoE (6.5B active), vLLM on ana-ml2 GPU 0 (dedicated 96 GB Blackwell), :8010.
|
||||
# Explicit entry shadows the "*" wildcard. TEXT-ONLY for now — vLLM 0.23.0's
|
||||
# Mistral multimodal processor crashes at startup (loaded with image/video
|
||||
# limit 0); vision returns when vLLM patches it. Deployed 2026-06-15. ---
|
||||
- model_name: mistral-small-4
|
||||
# --- Qwen3.5-122B-A10B (abliterated NVFP4, text-only) — the general / `gen` model.
|
||||
# REPLACED mistral-small-4 on ana-ml2 GPU 0 (operator 2026-06-19; heretic downed,
|
||||
# mistral entries REMOVED). bjk110 NVFP4 served on :8013 via vLLM + the text-only
|
||||
# patch, served-name qwen3.5-122-a10b. Thinking split = chat_template_kwargs.
|
||||
# enable_thinking + --reasoning-parser qwen3 (mirrors qwen3.6-35b-a3b). One upstream
|
||||
# fanned out under qwen3.5-122-a10b[-reasoning] + aliases qwen-large[-reasoning] +
|
||||
# gen[-reasoning]; the -reasoning variants enable thinking. Keep api_base in sync.
|
||||
# ⚠️ Worldtree CHARACTER backend (was bound to mistral-small-4) is dark until
|
||||
# repointed — operator-acknowledged. ---
|
||||
- model_name: qwen3.5-122-a10b
|
||||
litellm_params:
|
||||
model: hosted_vllm/mistral-small-4
|
||||
api_base: http://10.250.50.54:8010/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
model_info:
|
||||
mode: chat
|
||||
|
||||
# mistral-small-4-reasoning: same upstream checkpoint, reasoning ON (2026-06-15,
|
||||
# operator wanted "medium" — but Mistral's reasoning_effort is BINARY, only
|
||||
# 'none' or 'high' (a medium/low request 400s). 'high' is the sole reasoning-ON
|
||||
# level, so it carries the -reasoning intent. LiteLLM forwards extra_body to
|
||||
# vLLM; the mistral reasoning-parser splits [THINK]…[/THINK] into reasoning_content.
|
||||
- model_name: mistral-small-4-reasoning
|
||||
litellm_params:
|
||||
model: hosted_vllm/mistral-small-4
|
||||
api_base: http://10.250.50.54:8010/v1
|
||||
model: hosted_vllm/qwen3.5-122-a10b
|
||||
api_base: http://10.250.50.54:8013/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
extra_body:
|
||||
reasoning_effort: high
|
||||
chat_template_kwargs:
|
||||
enable_thinking: false
|
||||
model_info:
|
||||
mode: chat
|
||||
- model_name: qwen3.5-122-a10b-reasoning
|
||||
litellm_params:
|
||||
model: hosted_vllm/qwen3.5-122-a10b
|
||||
api_base: http://10.250.50.54:8013/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
extra_body:
|
||||
chat_template_kwargs:
|
||||
enable_thinking: true
|
||||
model_info:
|
||||
mode: chat
|
||||
- model_name: qwen-large
|
||||
litellm_params:
|
||||
model: hosted_vllm/qwen3.5-122-a10b
|
||||
api_base: http://10.250.50.54:8013/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
extra_body:
|
||||
chat_template_kwargs:
|
||||
enable_thinking: false
|
||||
model_info:
|
||||
mode: chat
|
||||
- model_name: qwen-large-reasoning
|
||||
litellm_params:
|
||||
model: hosted_vllm/qwen3.5-122-a10b
|
||||
api_base: http://10.250.50.54:8013/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
extra_body:
|
||||
chat_template_kwargs:
|
||||
enable_thinking: true
|
||||
model_info:
|
||||
mode: chat
|
||||
|
||||
# aliases: gen -> mistral-small-4, gen-reasoning -> mistral-small-4-reasoning
|
||||
# (operator 2026-06-19). Duplicate-entry aliases — keep api_base in sync with the
|
||||
# two mistral-small-4 entries above.
|
||||
- model_name: gen
|
||||
litellm_params:
|
||||
model: hosted_vllm/mistral-small-4
|
||||
api_base: http://10.250.50.54:8010/v1
|
||||
model: hosted_vllm/qwen3.5-122-a10b
|
||||
api_base: http://10.250.50.54:8013/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
extra_body:
|
||||
chat_template_kwargs:
|
||||
enable_thinking: false
|
||||
model_info:
|
||||
mode: chat
|
||||
- model_name: gen-reasoning
|
||||
litellm_params:
|
||||
model: hosted_vllm/mistral-small-4
|
||||
api_base: http://10.250.50.54:8010/v1
|
||||
model: hosted_vllm/qwen3.5-122-a10b
|
||||
api_base: http://10.250.50.54:8013/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
extra_body:
|
||||
reasoning_effort: high
|
||||
chat_template_kwargs:
|
||||
enable_thinking: true
|
||||
model_info:
|
||||
mode: chat
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# qwen3.5-122b (bjk110/Qwen3.5-122B-A10B-abliterated-NVFP4, text-only) — ana-ml2
|
||||
# GPU 0 tunables. Real .env lives at /opt/docker/compose/qwen3.5-122b/.env.
|
||||
|
||||
# vllm/vllm-openai:latest per the repo (the text-only patch targets latest).
|
||||
# MUTABLE tag — pin a digest once a known-good version is established.
|
||||
QWEN35_IMAGE=vllm/vllm-openai:latest
|
||||
QWEN35_CONTAINER_NAME=vllm-qwen35-122b
|
||||
|
||||
# Own port (8010=mistral heretic [downed], 8007=qwen36, 8011=selene — 8013 free).
|
||||
QWEN35_PORT=8013
|
||||
QWEN35_GPU_ID=0
|
||||
|
||||
# vLLM served-model-name; litellm fans out qwen3.5-122-a10b[-reasoning] + aliases
|
||||
# (qwen-large, gen) onto this, differentiated by chat_template_kwargs.enable_thinking.
|
||||
QWEN35_SERVED_NAME=qwen3.5-122-a10b
|
||||
|
||||
QWEN35_MAX_MODEL_LEN=131072
|
||||
QWEN35_MAX_NUM_SEQS=4
|
||||
QWEN35_GPU_MEM_UTIL=0.90
|
||||
QWEN35_MAX_NUM_BATCHED_TOKENS=32768
|
||||
|
||||
# Optional upstream vLLM API key (empty = no auth; internal net only).
|
||||
API_KEY=
|
||||
@@ -0,0 +1,81 @@
|
||||
# qwen3.5-122b — bjk110/Qwen3.5-122B-A10B-abliterated-NVFP4 on ana-ml2 GPU 0,
|
||||
# REPLACING mistral-small-4 (operator 2026-06-19: down the heretic, serve this as
|
||||
# the new general/`gen` model). Abliterated Qwen3.5 MoE (256 experts, 10B active),
|
||||
# NVFP4 (compressed-tensors), HF format.
|
||||
#
|
||||
# SERVING (per the repo's serving/): Qwen3.5 MoE is a MULTIMODAL arch but this
|
||||
# checkpoint is text-only, so vLLM needs the repo's text-only PATCH applied before
|
||||
# startup. We reuse the repo's entrypoint.sh (applies the patch, then runs vLLM) and
|
||||
# vllm_patches/, mounted from the downloaded model dir — keeps patch+checkpoint
|
||||
# version-coupled. Thinking split via litellm extra_body chat_template_kwargs
|
||||
# (enable_thinking) + --reasoning-parser qwen3 (mirrors the qwen3.6-35b-a3b pattern).
|
||||
#
|
||||
# DISPLACEMENT: GPU 0 fits one mistral-class model; bring this up only after downing
|
||||
# mistral-small-4-heretic. REVERT = down this, `up -d` the heretic stack.
|
||||
#
|
||||
# NOTE: image is vllm/vllm-openai:latest per the repo (the patch targets latest) —
|
||||
# MUTABLE tag; pin a digest once a known-good version is established.
|
||||
#
|
||||
# Tunables in .env.
|
||||
|
||||
name: qwen3.5-122b
|
||||
|
||||
services:
|
||||
vllm-qwen35-122b:
|
||||
image: ${QWEN35_IMAGE:-vllm/vllm-openai:latest}
|
||||
container_name: ${QWEN35_CONTAINER_NAME:-vllm-qwen35-122b}
|
||||
restart: unless-stopped
|
||||
ipc: host
|
||||
ports:
|
||||
- "${QWEN35_PORT:-8013}:8000"
|
||||
environment:
|
||||
- ROLE=head
|
||||
- TP_SIZE=1
|
||||
- MODEL_CONTAINER_PATH=/models/qwen
|
||||
- SERVED_MODEL_NAME=${QWEN35_SERVED_NAME:-qwen3.5-122-a10b}
|
||||
- HOST_PORT=8000
|
||||
- MAX_MODEL_LEN=${QWEN35_MAX_MODEL_LEN:-131072}
|
||||
- MAX_NUM_SEQS=${QWEN35_MAX_NUM_SEQS:-4}
|
||||
- GPU_MEMORY_UTILIZATION=${QWEN35_GPU_MEM_UTIL:-0.90}
|
||||
- MAX_NUM_BATCHED_TOKENS=${QWEN35_MAX_NUM_BATCHED_TOKENS:-32768}
|
||||
# --reasoning-parser qwen3 surfaces <think>…</think> as reasoning_content;
|
||||
# the thinking on/off itself is per-request (litellm chat_template_kwargs).
|
||||
- VLLM_EXTRA_ARGS=--reasoning-parser qwen3 --enable-chunked-prefill
|
||||
- NVIDIA_VISIBLE_DEVICES=0
|
||||
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
- VLLM_API_KEY=${API_KEY:-}
|
||||
volumes:
|
||||
# checkpoint + the repo's entrypoint/patch (downloaded with the model).
|
||||
- /tank/aimodels/qwen3.5-122b-a10b-nvfp4:/models/qwen:ro
|
||||
- /tank/aimodels/qwen3.5-122b-a10b-nvfp4/serving/entrypoint.sh:/entrypoint.sh:ro
|
||||
- /tank/aimodels/qwen3.5-122b-a10b-nvfp4/vllm_patches:/patches:ro
|
||||
- /tank/aimodels/qwen3.5-122b-a10b-nvfp4/.cache/vllm:/root/.cache/vllm
|
||||
entrypoint: ["bash", "/entrypoint.sh"]
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
device_ids:
|
||||
- "${QWEN35_GPU_ID:-0}"
|
||||
capabilities:
|
||||
- gpu
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 900s
|
||||
networks:
|
||||
- tnet
|
||||
labels:
|
||||
- homepage.group=AI Systems
|
||||
- homepage.name=Qwen3.5-122B-A10B (abliterated NVFP4)
|
||||
- homepage.icon=mdi-creation
|
||||
- homepage.description=Abliterated Qwen3.5 122B-A10B NVFP4, the new `gen` model (ana-ml2 GPU 0)
|
||||
- homepage.href=http://10.250.50.54:${QWEN35_PORT:-8013}/docs
|
||||
|
||||
networks:
|
||||
tnet:
|
||||
name: traefik-net
|
||||
external: true
|
||||
Reference in New Issue
Block a user